feat(attachment): upload files in resumable chunks

Add UploadAttachment, a unary RPC that accepts a file in bounded chunks
and streams it to a temp file instead of holding the whole blob in one
request body and in memory. A call carrying a spec starts an upload and
returns an opaque upload_id; later calls carry that id with
write_offset,
data, and finish_write, following the Cloud Storage WriteObject shape.
Retrying the last chunk after a lost response is idempotent, uploads are
bound to their owner, and they expire after 30 minutes of inactivity.

CreateAttachment and the chunked finalize now share one processing
pipeline for motion-photo detection, EXIF stripping, and storage. The
motion-photo detector works over an io.ReaderAt so large JPEGs are never
loaded whole, and the request body cap is a single per-procedure lookup.

The web editor uploads through the new RPC in 2 MiB chunks with bounded
concurrency under the server's per-user limit.
This commit is contained in:
boojack 2026-09-08 21:07:06 +08:00
parent 805e608aa5
commit b3e67399b8
27 changed files with 2185 additions and 223 deletions

View file

@ -20,25 +20,11 @@ var (
const maxMetadataScanBytes = 256 * 1024
// DetectJPEG detects embedded video in an in-memory JPEG.
func DetectJPEG(blob []byte) *Detection {
if len(blob) < 16 || !bytes.HasPrefix(blob, []byte{0xFF, 0xD8}) {
return nil
}
text := string(blob[:min(len(blob), maxMetadataScanBytes)])
if !motionPhotoMarkerRegex.MatchString(text) {
return nil
}
videoStart := detectVideoStart(blob, text)
if videoStart < 0 || videoStart >= len(blob) {
return nil
}
return &Detection{
VideoStart: videoStart,
PresentationTimestampUs: parsePresentationTimestampUs(text),
}
// bytes.Reader never fails within bounds, so the error is unreachable.
detection, _ := DetectJPEGReader(bytes.NewReader(blob), int64(len(blob)))
return detection
}
func ExtractVideo(blob []byte) ([]byte, *Detection) {
@ -55,19 +41,6 @@ func ExtractVideo(blob []byte) ([]byte, *Detection) {
return videoBlob, detection
}
func detectVideoStart(blob []byte, text string) int {
if matches := microVideoOffsetRegex.FindStringSubmatch(text); len(matches) == 2 {
if offset, err := strconv.Atoi(matches[1]); err == nil && offset > 0 && offset < len(blob) {
start := len(blob) - offset
if looksLikeMP4(blob[start:]) {
return start
}
}
}
return findEmbeddedMP4Start(blob)
}
func parsePresentationTimestampUs(text string) int64 {
matches := presentationRegex.FindStringSubmatch(text)
if len(matches) != 2 {

View file

@ -0,0 +1,56 @@
package motionphoto
import (
"bytes"
"io"
"strconv"
)
// DetectJPEGReader detects embedded video without loading the JPEG into memory.
// It reads at most the metadata header plus the blocks needed to locate the video.
func DetectJPEGReader(reader io.ReaderAt, size int64) (*Detection, error) {
if size < 16 {
return nil, nil
}
header := make([]byte, min(size, maxMetadataScanBytes))
if _, err := reader.ReadAt(header, 0); err != nil {
return nil, err
}
if !bytes.HasPrefix(header, []byte{0xFF, 0xD8}) || !motionPhotoMarkerRegex.Match(header) {
return nil, nil
}
text := string(header)
makeDetection := func(start int64) *Detection {
return &Detection{VideoStart: int(start), PresentationTimestampUs: parsePresentationTimestampUs(text)}
}
if matches := microVideoOffsetRegex.FindStringSubmatch(text); len(matches) == 2 {
if offset, err := strconv.ParseInt(matches[1], 10, 64); err == nil && offset >= 12 && offset < size {
var probe [12]byte
if _, err := reader.ReadAt(probe[:], size-offset); err != nil {
return nil, err
}
if looksLikeMP4(probe[:]) {
return makeDetection(size - offset), nil
}
}
}
// Fall back to the last valid ftyp box, scanning backwards with overlap so
// a box header crossing a block boundary is still found.
buffer := make([]byte, 64*1024)
for end := size; end >= 12; {
start := max(int64(0), end-int64(len(buffer)))
block := buffer[:end-start]
if _, err := reader.ReadAt(block, start); err != nil {
return nil, err
}
if index := findEmbeddedMP4Start(block); index >= 0 {
return makeDetection(start + int64(index)), nil
}
if start == 0 {
break
}
end = start + 11
}
return nil, nil
}

View file

@ -0,0 +1,45 @@
package motionphoto
import (
"bytes"
"testing"
"github.com/stretchr/testify/require"
"github.com/usememos/memos/internal/testutil"
)
func TestDetectJPEGReader(t *testing.T) {
for _, blob := range [][]byte{nil, []byte("not a JPEG"), []byte("\xff\xd8 no motion marker here at all")} {
detected, err := DetectJPEGReader(bytes.NewReader(blob), int64(len(blob)))
require.NoError(t, err)
require.Nil(t, detected)
}
fixture := testutil.BuildMotionPhotoJPEG()
detected, err := DetectJPEGReader(bytes.NewReader(fixture), int64(len(fixture)))
require.NoError(t, err)
require.NotNil(t, detected)
require.True(t, looksLikeMP4(fixture[detected.VideoStart:]))
require.EqualValues(t, 123456, detected.PresentationTimestampUs)
// Exercise MP4 headers straddling both sides of the backwards scan boundary.
for offset := -12; offset <= 12; offset++ {
blob := make([]byte, 200_000)
copy(blob, []byte("\xff\xd8 Camera:MotionPhoto=\"1\""))
start := len(blob) - 64*1024 + offset
copy(blob[start:], []byte{0, 0, 0, 16, 'f', 't', 'y', 'p', 'm', 'p', '4', '2'})
detected, err := DetectJPEGReader(bytes.NewReader(blob), int64(len(blob)))
require.NoError(t, err)
require.NotNil(t, detected, "offset %d", offset)
require.Equal(t, start, detected.VideoStart, "offset %d", offset)
}
// Prefer the explicit MicroVideoOffset even when a later ftyp box exists.
blob := make([]byte, 1024)
copy(blob, []byte("\xff\xd8 Camera:MotionPhoto=\"1\" Camera:MicroVideoOffset=\"128\""))
copy(blob[896:], []byte{0, 0, 0, 16, 'f', 't', 'y', 'p', 'm', 'p', '4', '2'})
copy(blob[1000:], []byte{0, 0, 0, 16, 'f', 't', 'y', 'p', 'm', 'p', '4', '2'})
detected, err = DetectJPEGReader(bytes.NewReader(blob), int64(len(blob)))
require.NoError(t, err)
require.Equal(t, 896, detected.VideoStart)
}

View file

@ -21,6 +21,16 @@ service AttachmentService {
};
option (google.api.method_signature) = "attachment";
}
// UploadAttachment uploads a file in bounded chunks. The first call carries
// the spec and returns an upload_id; later calls carry that upload_id.
// Uploads are bound to the authenticated user, expire after 30 minutes of
// inactivity, and do not survive a server restart.
rpc UploadAttachment(UploadAttachmentRequest) returns (UploadAttachmentResponse) {
option (google.api.http) = {
post: "/api/v1/attachments:upload"
body: "*"
};
}
// ListAttachments lists all attachments.
rpc ListAttachments(ListAttachmentsRequest) returns (ListAttachmentsResponse) {
option (google.api.http) = {get: "/api/v1/attachments"};
@ -179,6 +189,61 @@ message CreateAttachmentRequest {
string attachment_id = 2 [(google.api.field_behavior) = OPTIONAL];
}
message UploadAttachmentRequest {
// Required. Start a new upload or continue an existing one.
oneof upload {
// Starts a new upload. The same call may also carry data and finish_write.
UploadAttachmentSpec spec = 1;
// Continues the upload identified by a previous response.
string upload_id = 2;
}
// Required. Zero-based byte offset at which data is written. Must equal the
// committed size, except when retrying the most recently accepted chunk
// with identical bytes and offset, which is accepted without writing again.
int64 write_offset = 3 [(google.api.field_behavior) = REQUIRED];
// Optional. File bytes, at most max_chunk_size long. With no data and
// finish_write false, the call reports progress without writing, and
// write_offset is ignored.
bytes data = 4 [(google.api.field_behavior) = OPTIONAL];
// Optional. Finalize the upload after writing data. The committed size must
// then equal total_size. Any later call for the same upload_id returns the
// created attachment.
bool finish_write = 5 [(google.api.field_behavior) = OPTIONAL];
}
message UploadAttachmentSpec {
// Required. Metadata for the attachment to create. content must be empty;
// file bytes are sent in data.
Attachment attachment = 1 [(google.api.field_behavior) = REQUIRED];
// Optional. The attachment ID to use for this attachment.
// If empty, a unique ID will be generated.
// Format: ^[a-zA-Z0-9]([a-zA-Z0-9-]{0,34}[a-zA-Z0-9])?$
string attachment_id = 2 [(google.api.field_behavior) = OPTIONAL];
// Optional. Total size of the file in bytes before media processing.
// Zero represents an empty file.
int64 total_size = 3 [(google.api.field_behavior) = OPTIONAL];
}
message UploadAttachmentResponse {
// Opaque ID for subsequent calls. This is not a resource name.
string upload_id = 1;
// Number of file bytes committed so far.
int64 committed_size = 2;
// Set once the upload has been finalized.
Attachment attachment = 3;
// Maximum number of data bytes accepted in one call.
int32 max_chunk_size = 4;
}
message ListAttachmentsRequest {
// Optional. The maximum number of attachments to return.
// The service may return fewer than this value.

View file

@ -37,6 +37,9 @@ const (
// AttachmentServiceCreateAttachmentProcedure is the fully-qualified name of the AttachmentService's
// CreateAttachment RPC.
AttachmentServiceCreateAttachmentProcedure = "/memos.api.v1.AttachmentService/CreateAttachment"
// AttachmentServiceUploadAttachmentProcedure is the fully-qualified name of the AttachmentService's
// UploadAttachment RPC.
AttachmentServiceUploadAttachmentProcedure = "/memos.api.v1.AttachmentService/UploadAttachment"
// AttachmentServiceListAttachmentsProcedure is the fully-qualified name of the AttachmentService's
// ListAttachments RPC.
AttachmentServiceListAttachmentsProcedure = "/memos.api.v1.AttachmentService/ListAttachments"
@ -58,6 +61,11 @@ const (
type AttachmentServiceClient interface {
// CreateAttachment creates a new attachment.
CreateAttachment(context.Context, *connect.Request[v1.CreateAttachmentRequest]) (*connect.Response[v1.Attachment], error)
// UploadAttachment uploads a file in bounded chunks. The first call carries
// the spec and returns an upload_id; later calls carry that upload_id.
// Uploads are bound to the authenticated user, expire after 30 minutes of
// inactivity, and do not survive a server restart.
UploadAttachment(context.Context, *connect.Request[v1.UploadAttachmentRequest]) (*connect.Response[v1.UploadAttachmentResponse], error)
// ListAttachments lists all attachments.
ListAttachments(context.Context, *connect.Request[v1.ListAttachmentsRequest]) (*connect.Response[v1.ListAttachmentsResponse], error)
// GetAttachment returns an attachment by name.
@ -87,6 +95,12 @@ func NewAttachmentServiceClient(httpClient connect.HTTPClient, baseURL string, o
connect.WithSchema(attachmentServiceMethods.ByName("CreateAttachment")),
connect.WithClientOptions(opts...),
),
uploadAttachment: connect.NewClient[v1.UploadAttachmentRequest, v1.UploadAttachmentResponse](
httpClient,
baseURL+AttachmentServiceUploadAttachmentProcedure,
connect.WithSchema(attachmentServiceMethods.ByName("UploadAttachment")),
connect.WithClientOptions(opts...),
),
listAttachments: connect.NewClient[v1.ListAttachmentsRequest, v1.ListAttachmentsResponse](
httpClient,
baseURL+AttachmentServiceListAttachmentsProcedure,
@ -123,6 +137,7 @@ func NewAttachmentServiceClient(httpClient connect.HTTPClient, baseURL string, o
// attachmentServiceClient implements AttachmentServiceClient.
type attachmentServiceClient struct {
createAttachment *connect.Client[v1.CreateAttachmentRequest, v1.Attachment]
uploadAttachment *connect.Client[v1.UploadAttachmentRequest, v1.UploadAttachmentResponse]
listAttachments *connect.Client[v1.ListAttachmentsRequest, v1.ListAttachmentsResponse]
getAttachment *connect.Client[v1.GetAttachmentRequest, v1.Attachment]
updateAttachment *connect.Client[v1.UpdateAttachmentRequest, v1.Attachment]
@ -135,6 +150,11 @@ func (c *attachmentServiceClient) CreateAttachment(ctx context.Context, req *con
return c.createAttachment.CallUnary(ctx, req)
}
// UploadAttachment calls memos.api.v1.AttachmentService.UploadAttachment.
func (c *attachmentServiceClient) UploadAttachment(ctx context.Context, req *connect.Request[v1.UploadAttachmentRequest]) (*connect.Response[v1.UploadAttachmentResponse], error) {
return c.uploadAttachment.CallUnary(ctx, req)
}
// ListAttachments calls memos.api.v1.AttachmentService.ListAttachments.
func (c *attachmentServiceClient) ListAttachments(ctx context.Context, req *connect.Request[v1.ListAttachmentsRequest]) (*connect.Response[v1.ListAttachmentsResponse], error) {
return c.listAttachments.CallUnary(ctx, req)
@ -164,6 +184,11 @@ func (c *attachmentServiceClient) BatchDeleteAttachments(ctx context.Context, re
type AttachmentServiceHandler interface {
// CreateAttachment creates a new attachment.
CreateAttachment(context.Context, *connect.Request[v1.CreateAttachmentRequest]) (*connect.Response[v1.Attachment], error)
// UploadAttachment uploads a file in bounded chunks. The first call carries
// the spec and returns an upload_id; later calls carry that upload_id.
// Uploads are bound to the authenticated user, expire after 30 minutes of
// inactivity, and do not survive a server restart.
UploadAttachment(context.Context, *connect.Request[v1.UploadAttachmentRequest]) (*connect.Response[v1.UploadAttachmentResponse], error)
// ListAttachments lists all attachments.
ListAttachments(context.Context, *connect.Request[v1.ListAttachmentsRequest]) (*connect.Response[v1.ListAttachmentsResponse], error)
// GetAttachment returns an attachment by name.
@ -189,6 +214,12 @@ func NewAttachmentServiceHandler(svc AttachmentServiceHandler, opts ...connect.H
connect.WithSchema(attachmentServiceMethods.ByName("CreateAttachment")),
connect.WithHandlerOptions(opts...),
)
attachmentServiceUploadAttachmentHandler := connect.NewUnaryHandler(
AttachmentServiceUploadAttachmentProcedure,
svc.UploadAttachment,
connect.WithSchema(attachmentServiceMethods.ByName("UploadAttachment")),
connect.WithHandlerOptions(opts...),
)
attachmentServiceListAttachmentsHandler := connect.NewUnaryHandler(
AttachmentServiceListAttachmentsProcedure,
svc.ListAttachments,
@ -223,6 +254,8 @@ func NewAttachmentServiceHandler(svc AttachmentServiceHandler, opts ...connect.H
switch r.URL.Path {
case AttachmentServiceCreateAttachmentProcedure:
attachmentServiceCreateAttachmentHandler.ServeHTTP(w, r)
case AttachmentServiceUploadAttachmentProcedure:
attachmentServiceUploadAttachmentHandler.ServeHTTP(w, r)
case AttachmentServiceListAttachmentsProcedure:
attachmentServiceListAttachmentsHandler.ServeHTTP(w, r)
case AttachmentServiceGetAttachmentProcedure:
@ -246,6 +279,10 @@ func (UnimplementedAttachmentServiceHandler) CreateAttachment(context.Context, *
return nil, connect.NewError(connect.CodeUnimplemented, errors.New("memos.api.v1.AttachmentService.CreateAttachment is not implemented"))
}
func (UnimplementedAttachmentServiceHandler) UploadAttachment(context.Context, *connect.Request[v1.UploadAttachmentRequest]) (*connect.Response[v1.UploadAttachmentResponse], error) {
return nil, connect.NewError(connect.CodeUnimplemented, errors.New("memos.api.v1.AttachmentService.UploadAttachment is not implemented"))
}
func (UnimplementedAttachmentServiceHandler) ListAttachments(context.Context, *connect.Request[v1.ListAttachmentsRequest]) (*connect.Response[v1.ListAttachmentsResponse], error) {
return nil, connect.NewError(connect.CodeUnimplemented, errors.New("memos.api.v1.AttachmentService.ListAttachments is not implemented"))
}

View file

@ -770,6 +770,264 @@ func (x *CreateAttachmentRequest) GetAttachmentId() string {
return ""
}
type UploadAttachmentRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
// Required. Start a new upload or continue an existing one.
//
// Types that are valid to be assigned to Upload:
//
// *UploadAttachmentRequest_Spec
// *UploadAttachmentRequest_UploadId
Upload isUploadAttachmentRequest_Upload `protobuf_oneof:"upload"`
// Required. Zero-based byte offset at which data is written. Must equal the
// committed size, except when retrying the most recently accepted chunk
// with identical bytes and offset, which is accepted without writing again.
WriteOffset int64 `protobuf:"varint,3,opt,name=write_offset,json=writeOffset,proto3" json:"write_offset,omitempty"`
// Optional. File bytes, at most max_chunk_size long. With no data and
// finish_write false, the call reports progress without writing, and
// write_offset is ignored.
Data []byte `protobuf:"bytes,4,opt,name=data,proto3" json:"data,omitempty"`
// Optional. Finalize the upload after writing data. The committed size must
// then equal total_size. Any later call for the same upload_id returns the
// created attachment.
FinishWrite bool `protobuf:"varint,5,opt,name=finish_write,json=finishWrite,proto3" json:"finish_write,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *UploadAttachmentRequest) Reset() {
*x = UploadAttachmentRequest{}
mi := &file_api_v1_attachment_service_proto_msgTypes[8]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *UploadAttachmentRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*UploadAttachmentRequest) ProtoMessage() {}
func (x *UploadAttachmentRequest) ProtoReflect() protoreflect.Message {
mi := &file_api_v1_attachment_service_proto_msgTypes[8]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use UploadAttachmentRequest.ProtoReflect.Descriptor instead.
func (*UploadAttachmentRequest) Descriptor() ([]byte, []int) {
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{8}
}
func (x *UploadAttachmentRequest) GetUpload() isUploadAttachmentRequest_Upload {
if x != nil {
return x.Upload
}
return nil
}
func (x *UploadAttachmentRequest) GetSpec() *UploadAttachmentSpec {
if x != nil {
if x, ok := x.Upload.(*UploadAttachmentRequest_Spec); ok {
return x.Spec
}
}
return nil
}
func (x *UploadAttachmentRequest) GetUploadId() string {
if x != nil {
if x, ok := x.Upload.(*UploadAttachmentRequest_UploadId); ok {
return x.UploadId
}
}
return ""
}
func (x *UploadAttachmentRequest) GetWriteOffset() int64 {
if x != nil {
return x.WriteOffset
}
return 0
}
func (x *UploadAttachmentRequest) GetData() []byte {
if x != nil {
return x.Data
}
return nil
}
func (x *UploadAttachmentRequest) GetFinishWrite() bool {
if x != nil {
return x.FinishWrite
}
return false
}
type isUploadAttachmentRequest_Upload interface {
isUploadAttachmentRequest_Upload()
}
type UploadAttachmentRequest_Spec struct {
// Starts a new upload. The same call may also carry data and finish_write.
Spec *UploadAttachmentSpec `protobuf:"bytes,1,opt,name=spec,proto3,oneof"`
}
type UploadAttachmentRequest_UploadId struct {
// Continues the upload identified by a previous response.
UploadId string `protobuf:"bytes,2,opt,name=upload_id,json=uploadId,proto3,oneof"`
}
func (*UploadAttachmentRequest_Spec) isUploadAttachmentRequest_Upload() {}
func (*UploadAttachmentRequest_UploadId) isUploadAttachmentRequest_Upload() {}
type UploadAttachmentSpec struct {
state protoimpl.MessageState `protogen:"open.v1"`
// Required. Metadata for the attachment to create. content must be empty;
// file bytes are sent in data.
Attachment *Attachment `protobuf:"bytes,1,opt,name=attachment,proto3" json:"attachment,omitempty"`
// Optional. The attachment ID to use for this attachment.
// If empty, a unique ID will be generated.
// Format: ^[a-zA-Z0-9]([a-zA-Z0-9-]{0,34}[a-zA-Z0-9])?$
AttachmentId string `protobuf:"bytes,2,opt,name=attachment_id,json=attachmentId,proto3" json:"attachment_id,omitempty"`
// Optional. Total size of the file in bytes before media processing.
// Zero represents an empty file.
TotalSize int64 `protobuf:"varint,3,opt,name=total_size,json=totalSize,proto3" json:"total_size,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *UploadAttachmentSpec) Reset() {
*x = UploadAttachmentSpec{}
mi := &file_api_v1_attachment_service_proto_msgTypes[9]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *UploadAttachmentSpec) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*UploadAttachmentSpec) ProtoMessage() {}
func (x *UploadAttachmentSpec) ProtoReflect() protoreflect.Message {
mi := &file_api_v1_attachment_service_proto_msgTypes[9]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use UploadAttachmentSpec.ProtoReflect.Descriptor instead.
func (*UploadAttachmentSpec) Descriptor() ([]byte, []int) {
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{9}
}
func (x *UploadAttachmentSpec) GetAttachment() *Attachment {
if x != nil {
return x.Attachment
}
return nil
}
func (x *UploadAttachmentSpec) GetAttachmentId() string {
if x != nil {
return x.AttachmentId
}
return ""
}
func (x *UploadAttachmentSpec) GetTotalSize() int64 {
if x != nil {
return x.TotalSize
}
return 0
}
type UploadAttachmentResponse struct {
state protoimpl.MessageState `protogen:"open.v1"`
// Opaque ID for subsequent calls. This is not a resource name.
UploadId string `protobuf:"bytes,1,opt,name=upload_id,json=uploadId,proto3" json:"upload_id,omitempty"`
// Number of file bytes committed so far.
CommittedSize int64 `protobuf:"varint,2,opt,name=committed_size,json=committedSize,proto3" json:"committed_size,omitempty"`
// Set once the upload has been finalized.
Attachment *Attachment `protobuf:"bytes,3,opt,name=attachment,proto3" json:"attachment,omitempty"`
// Maximum number of data bytes accepted in one call.
MaxChunkSize int32 `protobuf:"varint,4,opt,name=max_chunk_size,json=maxChunkSize,proto3" json:"max_chunk_size,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *UploadAttachmentResponse) Reset() {
*x = UploadAttachmentResponse{}
mi := &file_api_v1_attachment_service_proto_msgTypes[10]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *UploadAttachmentResponse) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*UploadAttachmentResponse) ProtoMessage() {}
func (x *UploadAttachmentResponse) ProtoReflect() protoreflect.Message {
mi := &file_api_v1_attachment_service_proto_msgTypes[10]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use UploadAttachmentResponse.ProtoReflect.Descriptor instead.
func (*UploadAttachmentResponse) Descriptor() ([]byte, []int) {
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{10}
}
func (x *UploadAttachmentResponse) GetUploadId() string {
if x != nil {
return x.UploadId
}
return ""
}
func (x *UploadAttachmentResponse) GetCommittedSize() int64 {
if x != nil {
return x.CommittedSize
}
return 0
}
func (x *UploadAttachmentResponse) GetAttachment() *Attachment {
if x != nil {
return x.Attachment
}
return nil
}
func (x *UploadAttachmentResponse) GetMaxChunkSize() int32 {
if x != nil {
return x.MaxChunkSize
}
return 0
}
type ListAttachmentsRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
// Optional. The maximum number of attachments to return.
@ -798,7 +1056,7 @@ type ListAttachmentsRequest struct {
func (x *ListAttachmentsRequest) Reset() {
*x = ListAttachmentsRequest{}
mi := &file_api_v1_attachment_service_proto_msgTypes[8]
mi := &file_api_v1_attachment_service_proto_msgTypes[11]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@ -810,7 +1068,7 @@ func (x *ListAttachmentsRequest) String() string {
func (*ListAttachmentsRequest) ProtoMessage() {}
func (x *ListAttachmentsRequest) ProtoReflect() protoreflect.Message {
mi := &file_api_v1_attachment_service_proto_msgTypes[8]
mi := &file_api_v1_attachment_service_proto_msgTypes[11]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@ -823,7 +1081,7 @@ func (x *ListAttachmentsRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use ListAttachmentsRequest.ProtoReflect.Descriptor instead.
func (*ListAttachmentsRequest) Descriptor() ([]byte, []int) {
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{8}
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{11}
}
func (x *ListAttachmentsRequest) GetPageSize() int32 {
@ -867,7 +1125,7 @@ type ListAttachmentsResponse struct {
func (x *ListAttachmentsResponse) Reset() {
*x = ListAttachmentsResponse{}
mi := &file_api_v1_attachment_service_proto_msgTypes[9]
mi := &file_api_v1_attachment_service_proto_msgTypes[12]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@ -879,7 +1137,7 @@ func (x *ListAttachmentsResponse) String() string {
func (*ListAttachmentsResponse) ProtoMessage() {}
func (x *ListAttachmentsResponse) ProtoReflect() protoreflect.Message {
mi := &file_api_v1_attachment_service_proto_msgTypes[9]
mi := &file_api_v1_attachment_service_proto_msgTypes[12]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@ -892,7 +1150,7 @@ func (x *ListAttachmentsResponse) ProtoReflect() protoreflect.Message {
// Deprecated: Use ListAttachmentsResponse.ProtoReflect.Descriptor instead.
func (*ListAttachmentsResponse) Descriptor() ([]byte, []int) {
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{9}
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{12}
}
func (x *ListAttachmentsResponse) GetAttachments() []*Attachment {
@ -920,7 +1178,7 @@ type GetAttachmentRequest struct {
func (x *GetAttachmentRequest) Reset() {
*x = GetAttachmentRequest{}
mi := &file_api_v1_attachment_service_proto_msgTypes[10]
mi := &file_api_v1_attachment_service_proto_msgTypes[13]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@ -932,7 +1190,7 @@ func (x *GetAttachmentRequest) String() string {
func (*GetAttachmentRequest) ProtoMessage() {}
func (x *GetAttachmentRequest) ProtoReflect() protoreflect.Message {
mi := &file_api_v1_attachment_service_proto_msgTypes[10]
mi := &file_api_v1_attachment_service_proto_msgTypes[13]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@ -945,7 +1203,7 @@ func (x *GetAttachmentRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use GetAttachmentRequest.ProtoReflect.Descriptor instead.
func (*GetAttachmentRequest) Descriptor() ([]byte, []int) {
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{10}
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{13}
}
func (x *GetAttachmentRequest) GetName() string {
@ -967,7 +1225,7 @@ type UpdateAttachmentRequest struct {
func (x *UpdateAttachmentRequest) Reset() {
*x = UpdateAttachmentRequest{}
mi := &file_api_v1_attachment_service_proto_msgTypes[11]
mi := &file_api_v1_attachment_service_proto_msgTypes[14]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@ -979,7 +1237,7 @@ func (x *UpdateAttachmentRequest) String() string {
func (*UpdateAttachmentRequest) ProtoMessage() {}
func (x *UpdateAttachmentRequest) ProtoReflect() protoreflect.Message {
mi := &file_api_v1_attachment_service_proto_msgTypes[11]
mi := &file_api_v1_attachment_service_proto_msgTypes[14]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@ -992,7 +1250,7 @@ func (x *UpdateAttachmentRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use UpdateAttachmentRequest.ProtoReflect.Descriptor instead.
func (*UpdateAttachmentRequest) Descriptor() ([]byte, []int) {
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{11}
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{14}
}
func (x *UpdateAttachmentRequest) GetAttachment() *Attachment {
@ -1020,7 +1278,7 @@ type DeleteAttachmentRequest struct {
func (x *DeleteAttachmentRequest) Reset() {
*x = DeleteAttachmentRequest{}
mi := &file_api_v1_attachment_service_proto_msgTypes[12]
mi := &file_api_v1_attachment_service_proto_msgTypes[15]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@ -1032,7 +1290,7 @@ func (x *DeleteAttachmentRequest) String() string {
func (*DeleteAttachmentRequest) ProtoMessage() {}
func (x *DeleteAttachmentRequest) ProtoReflect() protoreflect.Message {
mi := &file_api_v1_attachment_service_proto_msgTypes[12]
mi := &file_api_v1_attachment_service_proto_msgTypes[15]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@ -1045,7 +1303,7 @@ func (x *DeleteAttachmentRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use DeleteAttachmentRequest.ProtoReflect.Descriptor instead.
func (*DeleteAttachmentRequest) Descriptor() ([]byte, []int) {
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{12}
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{15}
}
func (x *DeleteAttachmentRequest) GetName() string {
@ -1064,7 +1322,7 @@ type BatchDeleteAttachmentsRequest struct {
func (x *BatchDeleteAttachmentsRequest) Reset() {
*x = BatchDeleteAttachmentsRequest{}
mi := &file_api_v1_attachment_service_proto_msgTypes[13]
mi := &file_api_v1_attachment_service_proto_msgTypes[16]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@ -1076,7 +1334,7 @@ func (x *BatchDeleteAttachmentsRequest) String() string {
func (*BatchDeleteAttachmentsRequest) ProtoMessage() {}
func (x *BatchDeleteAttachmentsRequest) ProtoReflect() protoreflect.Message {
mi := &file_api_v1_attachment_service_proto_msgTypes[13]
mi := &file_api_v1_attachment_service_proto_msgTypes[16]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@ -1089,7 +1347,7 @@ func (x *BatchDeleteAttachmentsRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use BatchDeleteAttachmentsRequest.ProtoReflect.Descriptor instead.
func (*BatchDeleteAttachmentsRequest) Descriptor() ([]byte, []int) {
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{13}
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{16}
}
func (x *BatchDeleteAttachmentsRequest) GetNames() []string {
@ -1174,7 +1432,28 @@ const file_api_v1_attachment_service_proto_rawDesc = "" +
"\n" +
"attachment\x18\x01 \x01(\v2\x18.memos.api.v1.AttachmentB\x03\xe0A\x02R\n" +
"attachment\x12(\n" +
"\rattachment_id\x18\x02 \x01(\tB\x03\xe0A\x01R\fattachmentId\"\xba\x01\n" +
"\rattachment_id\x18\x02 \x01(\tB\x03\xe0A\x01R\fattachmentId\"\xe5\x01\n" +
"\x17UploadAttachmentRequest\x128\n" +
"\x04spec\x18\x01 \x01(\v2\".memos.api.v1.UploadAttachmentSpecH\x00R\x04spec\x12\x1d\n" +
"\tupload_id\x18\x02 \x01(\tH\x00R\buploadId\x12&\n" +
"\fwrite_offset\x18\x03 \x01(\x03B\x03\xe0A\x02R\vwriteOffset\x12\x17\n" +
"\x04data\x18\x04 \x01(\fB\x03\xe0A\x01R\x04data\x12&\n" +
"\ffinish_write\x18\x05 \x01(\bB\x03\xe0A\x01R\vfinishWriteB\b\n" +
"\x06upload\"\xa3\x01\n" +
"\x14UploadAttachmentSpec\x12=\n" +
"\n" +
"attachment\x18\x01 \x01(\v2\x18.memos.api.v1.AttachmentB\x03\xe0A\x02R\n" +
"attachment\x12(\n" +
"\rattachment_id\x18\x02 \x01(\tB\x03\xe0A\x01R\fattachmentId\x12\"\n" +
"\n" +
"total_size\x18\x03 \x01(\x03B\x03\xe0A\x01R\ttotalSize\"\xbe\x01\n" +
"\x18UploadAttachmentResponse\x12\x1b\n" +
"\tupload_id\x18\x01 \x01(\tR\buploadId\x12%\n" +
"\x0ecommitted_size\x18\x02 \x01(\x03R\rcommittedSize\x128\n" +
"\n" +
"attachment\x18\x03 \x01(\v2\x18.memos.api.v1.AttachmentR\n" +
"attachment\x12$\n" +
"\x0emax_chunk_size\x18\x04 \x01(\x05R\fmaxChunkSize\"\xba\x01\n" +
"\x16ListAttachmentsRequest\x12 \n" +
"\tpage_size\x18\x01 \x01(\x05B\x03\xe0A\x01R\bpageSize\x12\"\n" +
"\n" +
@ -1207,11 +1486,12 @@ const file_api_v1_attachment_service_proto_rawDesc = "" +
"\x1dMOTION_MEDIA_ROLE_UNSPECIFIED\x10\x00\x12\t\n" +
"\x05STILL\x10\x01\x12\t\n" +
"\x05VIDEO\x10\x02\x12\r\n" +
"\tCONTAINER\x10\x032\xd0\x06\n" +
"\tCONTAINER\x10\x032\xdb\a\n" +
"\x11AttachmentService\x12\x89\x01\n" +
"\x10CreateAttachment\x12%.memos.api.v1.CreateAttachmentRequest\x1a\x18.memos.api.v1.Attachment\"4\xdaA\n" +
"attachment\x82\xd3\xe4\x93\x02!:\n" +
"attachment\"\x13/api/v1/attachments\x12{\n" +
"attachment\"\x13/api/v1/attachments\x12\x88\x01\n" +
"\x10UploadAttachment\x12%.memos.api.v1.UploadAttachmentRequest\x1a&.memos.api.v1.UploadAttachmentResponse\"%\x82\xd3\xe4\x93\x02\x1f:\x01*\"\x1a/api/v1/attachments:upload\x12{\n" +
"\x0fListAttachments\x12$.memos.api.v1.ListAttachmentsRequest\x1a%.memos.api.v1.ListAttachmentsResponse\"\x1b\x82\xd3\xe4\x93\x02\x15\x12\x13/api/v1/attachments\x12z\n" +
"\rGetAttachment\x12\".memos.api.v1.GetAttachmentRequest\x1a\x18.memos.api.v1.Attachment\"+\xdaA\x04name\x82\xd3\xe4\x93\x02\x1e\x12\x1c/api/v1/{name=attachments/*}\x12\xa9\x01\n" +
"\x10UpdateAttachment\x12%.memos.api.v1.UpdateAttachmentRequest\x1a\x18.memos.api.v1.Attachment\"T\xdaA\x16attachment,update_mask\x82\xd3\xe4\x93\x025:\n" +
@ -1233,7 +1513,7 @@ func file_api_v1_attachment_service_proto_rawDescGZIP() []byte {
}
var file_api_v1_attachment_service_proto_enumTypes = make([]protoimpl.EnumInfo, 2)
var file_api_v1_attachment_service_proto_msgTypes = make([]protoimpl.MessageInfo, 14)
var file_api_v1_attachment_service_proto_msgTypes = make([]protoimpl.MessageInfo, 17)
var file_api_v1_attachment_service_proto_goTypes = []any{
(MotionMediaFamily)(0), // 0: memos.api.v1.MotionMediaFamily
(MotionMediaRole)(0), // 1: memos.api.v1.MotionMediaRole
@ -1245,15 +1525,18 @@ var file_api_v1_attachment_service_proto_goTypes = []any{
(*VideoMetadata)(nil), // 7: memos.api.v1.VideoMetadata
(*Attachment)(nil), // 8: memos.api.v1.Attachment
(*CreateAttachmentRequest)(nil), // 9: memos.api.v1.CreateAttachmentRequest
(*ListAttachmentsRequest)(nil), // 10: memos.api.v1.ListAttachmentsRequest
(*ListAttachmentsResponse)(nil), // 11: memos.api.v1.ListAttachmentsResponse
(*GetAttachmentRequest)(nil), // 12: memos.api.v1.GetAttachmentRequest
(*UpdateAttachmentRequest)(nil), // 13: memos.api.v1.UpdateAttachmentRequest
(*DeleteAttachmentRequest)(nil), // 14: memos.api.v1.DeleteAttachmentRequest
(*BatchDeleteAttachmentsRequest)(nil), // 15: memos.api.v1.BatchDeleteAttachmentsRequest
(*timestamppb.Timestamp)(nil), // 16: google.protobuf.Timestamp
(*fieldmaskpb.FieldMask)(nil), // 17: google.protobuf.FieldMask
(*emptypb.Empty)(nil), // 18: google.protobuf.Empty
(*UploadAttachmentRequest)(nil), // 10: memos.api.v1.UploadAttachmentRequest
(*UploadAttachmentSpec)(nil), // 11: memos.api.v1.UploadAttachmentSpec
(*UploadAttachmentResponse)(nil), // 12: memos.api.v1.UploadAttachmentResponse
(*ListAttachmentsRequest)(nil), // 13: memos.api.v1.ListAttachmentsRequest
(*ListAttachmentsResponse)(nil), // 14: memos.api.v1.ListAttachmentsResponse
(*GetAttachmentRequest)(nil), // 15: memos.api.v1.GetAttachmentRequest
(*UpdateAttachmentRequest)(nil), // 16: memos.api.v1.UpdateAttachmentRequest
(*DeleteAttachmentRequest)(nil), // 17: memos.api.v1.DeleteAttachmentRequest
(*BatchDeleteAttachmentsRequest)(nil), // 18: memos.api.v1.BatchDeleteAttachmentsRequest
(*timestamppb.Timestamp)(nil), // 19: google.protobuf.Timestamp
(*fieldmaskpb.FieldMask)(nil), // 20: google.protobuf.FieldMask
(*emptypb.Empty)(nil), // 21: google.protobuf.Empty
}
var file_api_v1_attachment_service_proto_depIdxs = []int32{
0, // 0: memos.api.v1.MotionMedia.family:type_name -> memos.api.v1.MotionMediaFamily
@ -1262,30 +1545,35 @@ var file_api_v1_attachment_service_proto_depIdxs = []int32{
7, // 3: memos.api.v1.MediaMetadata.video:type_name -> memos.api.v1.VideoMetadata
5, // 4: memos.api.v1.PhotoMetadata.capture_time:type_name -> memos.api.v1.MediaCaptureTime
6, // 5: memos.api.v1.PhotoMetadata.location:type_name -> memos.api.v1.MediaLocation
16, // 6: memos.api.v1.Attachment.create_time:type_name -> google.protobuf.Timestamp
19, // 6: memos.api.v1.Attachment.create_time:type_name -> google.protobuf.Timestamp
2, // 7: memos.api.v1.Attachment.motion_media:type_name -> memos.api.v1.MotionMedia
3, // 8: memos.api.v1.Attachment.media_metadata:type_name -> memos.api.v1.MediaMetadata
8, // 9: memos.api.v1.CreateAttachmentRequest.attachment:type_name -> memos.api.v1.Attachment
8, // 10: memos.api.v1.ListAttachmentsResponse.attachments:type_name -> memos.api.v1.Attachment
8, // 11: memos.api.v1.UpdateAttachmentRequest.attachment:type_name -> memos.api.v1.Attachment
17, // 12: memos.api.v1.UpdateAttachmentRequest.update_mask:type_name -> google.protobuf.FieldMask
9, // 13: memos.api.v1.AttachmentService.CreateAttachment:input_type -> memos.api.v1.CreateAttachmentRequest
10, // 14: memos.api.v1.AttachmentService.ListAttachments:input_type -> memos.api.v1.ListAttachmentsRequest
12, // 15: memos.api.v1.AttachmentService.GetAttachment:input_type -> memos.api.v1.GetAttachmentRequest
13, // 16: memos.api.v1.AttachmentService.UpdateAttachment:input_type -> memos.api.v1.UpdateAttachmentRequest
14, // 17: memos.api.v1.AttachmentService.DeleteAttachment:input_type -> memos.api.v1.DeleteAttachmentRequest
15, // 18: memos.api.v1.AttachmentService.BatchDeleteAttachments:input_type -> memos.api.v1.BatchDeleteAttachmentsRequest
8, // 19: memos.api.v1.AttachmentService.CreateAttachment:output_type -> memos.api.v1.Attachment
11, // 20: memos.api.v1.AttachmentService.ListAttachments:output_type -> memos.api.v1.ListAttachmentsResponse
8, // 21: memos.api.v1.AttachmentService.GetAttachment:output_type -> memos.api.v1.Attachment
8, // 22: memos.api.v1.AttachmentService.UpdateAttachment:output_type -> memos.api.v1.Attachment
18, // 23: memos.api.v1.AttachmentService.DeleteAttachment:output_type -> google.protobuf.Empty
18, // 24: memos.api.v1.AttachmentService.BatchDeleteAttachments:output_type -> google.protobuf.Empty
19, // [19:25] is the sub-list for method output_type
13, // [13:19] is the sub-list for method input_type
13, // [13:13] is the sub-list for extension type_name
13, // [13:13] is the sub-list for extension extendee
0, // [0:13] is the sub-list for field type_name
11, // 10: memos.api.v1.UploadAttachmentRequest.spec:type_name -> memos.api.v1.UploadAttachmentSpec
8, // 11: memos.api.v1.UploadAttachmentSpec.attachment:type_name -> memos.api.v1.Attachment
8, // 12: memos.api.v1.UploadAttachmentResponse.attachment:type_name -> memos.api.v1.Attachment
8, // 13: memos.api.v1.ListAttachmentsResponse.attachments:type_name -> memos.api.v1.Attachment
8, // 14: memos.api.v1.UpdateAttachmentRequest.attachment:type_name -> memos.api.v1.Attachment
20, // 15: memos.api.v1.UpdateAttachmentRequest.update_mask:type_name -> google.protobuf.FieldMask
9, // 16: memos.api.v1.AttachmentService.CreateAttachment:input_type -> memos.api.v1.CreateAttachmentRequest
10, // 17: memos.api.v1.AttachmentService.UploadAttachment:input_type -> memos.api.v1.UploadAttachmentRequest
13, // 18: memos.api.v1.AttachmentService.ListAttachments:input_type -> memos.api.v1.ListAttachmentsRequest
15, // 19: memos.api.v1.AttachmentService.GetAttachment:input_type -> memos.api.v1.GetAttachmentRequest
16, // 20: memos.api.v1.AttachmentService.UpdateAttachment:input_type -> memos.api.v1.UpdateAttachmentRequest
17, // 21: memos.api.v1.AttachmentService.DeleteAttachment:input_type -> memos.api.v1.DeleteAttachmentRequest
18, // 22: memos.api.v1.AttachmentService.BatchDeleteAttachments:input_type -> memos.api.v1.BatchDeleteAttachmentsRequest
8, // 23: memos.api.v1.AttachmentService.CreateAttachment:output_type -> memos.api.v1.Attachment
12, // 24: memos.api.v1.AttachmentService.UploadAttachment:output_type -> memos.api.v1.UploadAttachmentResponse
14, // 25: memos.api.v1.AttachmentService.ListAttachments:output_type -> memos.api.v1.ListAttachmentsResponse
8, // 26: memos.api.v1.AttachmentService.GetAttachment:output_type -> memos.api.v1.Attachment
8, // 27: memos.api.v1.AttachmentService.UpdateAttachment:output_type -> memos.api.v1.Attachment
21, // 28: memos.api.v1.AttachmentService.DeleteAttachment:output_type -> google.protobuf.Empty
21, // 29: memos.api.v1.AttachmentService.BatchDeleteAttachments:output_type -> google.protobuf.Empty
23, // [23:30] is the sub-list for method output_type
16, // [16:23] is the sub-list for method input_type
16, // [16:16] is the sub-list for extension type_name
16, // [16:16] is the sub-list for extension extendee
0, // [0:16] is the sub-list for field type_name
}
func init() { file_api_v1_attachment_service_proto_init() }
@ -1302,13 +1590,17 @@ func file_api_v1_attachment_service_proto_init() {
file_api_v1_attachment_service_proto_msgTypes[4].OneofWrappers = []any{}
file_api_v1_attachment_service_proto_msgTypes[5].OneofWrappers = []any{}
file_api_v1_attachment_service_proto_msgTypes[6].OneofWrappers = []any{}
file_api_v1_attachment_service_proto_msgTypes[8].OneofWrappers = []any{
(*UploadAttachmentRequest_Spec)(nil),
(*UploadAttachmentRequest_UploadId)(nil),
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_api_v1_attachment_service_proto_rawDesc), len(file_api_v1_attachment_service_proto_rawDesc)),
NumEnums: 2,
NumMessages: 14,
NumMessages: 17,
NumExtensions: 0,
NumServices: 1,
},

View file

@ -76,6 +76,33 @@ func local_request_AttachmentService_CreateAttachment_0(ctx context.Context, mar
return msg, metadata, err
}
func request_AttachmentService_UploadAttachment_0(ctx context.Context, marshaler runtime.Marshaler, client AttachmentServiceClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) {
var (
protoReq UploadAttachmentRequest
metadata runtime.ServerMetadata
)
if err := marshaler.NewDecoder(req.Body).Decode(&protoReq); err != nil && !errors.Is(err, io.EOF) {
return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err)
}
if req.Body != nil {
_, _ = io.Copy(io.Discard, req.Body)
}
msg, err := client.UploadAttachment(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD))
return msg, metadata, err
}
func local_request_AttachmentService_UploadAttachment_0(ctx context.Context, marshaler runtime.Marshaler, server AttachmentServiceServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) {
var (
protoReq UploadAttachmentRequest
metadata runtime.ServerMetadata
)
if err := marshaler.NewDecoder(req.Body).Decode(&protoReq); err != nil && !errors.Is(err, io.EOF) {
return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err)
}
msg, err := server.UploadAttachment(ctx, &protoReq)
return msg, metadata, err
}
var filter_AttachmentService_ListAttachments_0 = &utilities.DoubleArray{Encoding: map[string]int{}, Base: []int(nil), Check: []int(nil)}
func request_AttachmentService_ListAttachments_0(ctx context.Context, marshaler runtime.Marshaler, client AttachmentServiceClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) {
@ -323,6 +350,26 @@ func RegisterAttachmentServiceHandlerServer(ctx context.Context, mux *runtime.Se
}
forward_AttachmentService_CreateAttachment_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...)
})
mux.Handle(http.MethodPost, pattern_AttachmentService_UploadAttachment_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
ctx, cancel := context.WithCancel(req.Context())
defer cancel()
var stream runtime.ServerTransportStream
ctx = grpc.NewContextWithServerTransportStream(ctx, &stream)
inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req)
annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/memos.api.v1.AttachmentService/UploadAttachment", runtime.WithHTTPPathPattern("/api/v1/attachments:upload"))
if err != nil {
runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
return
}
resp, md, err := local_request_AttachmentService_UploadAttachment_0(annotatedContext, inboundMarshaler, server, req, pathParams)
md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer())
annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md)
if err != nil {
runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err)
return
}
forward_AttachmentService_UploadAttachment_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...)
})
mux.Handle(http.MethodGet, pattern_AttachmentService_ListAttachments_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
ctx, cancel := context.WithCancel(req.Context())
defer cancel()
@ -480,6 +527,23 @@ func RegisterAttachmentServiceHandlerClient(ctx context.Context, mux *runtime.Se
}
forward_AttachmentService_CreateAttachment_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...)
})
mux.Handle(http.MethodPost, pattern_AttachmentService_UploadAttachment_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
ctx, cancel := context.WithCancel(req.Context())
defer cancel()
inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req)
annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/memos.api.v1.AttachmentService/UploadAttachment", runtime.WithHTTPPathPattern("/api/v1/attachments:upload"))
if err != nil {
runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
return
}
resp, md, err := request_AttachmentService_UploadAttachment_0(annotatedContext, inboundMarshaler, client, req, pathParams)
annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md)
if err != nil {
runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err)
return
}
forward_AttachmentService_UploadAttachment_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...)
})
mux.Handle(http.MethodGet, pattern_AttachmentService_ListAttachments_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
ctx, cancel := context.WithCancel(req.Context())
defer cancel()
@ -570,6 +634,7 @@ func RegisterAttachmentServiceHandlerClient(ctx context.Context, mux *runtime.Se
var (
pattern_AttachmentService_CreateAttachment_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"api", "v1", "attachments"}, ""))
pattern_AttachmentService_UploadAttachment_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"api", "v1", "attachments"}, "upload"))
pattern_AttachmentService_ListAttachments_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"api", "v1", "attachments"}, ""))
pattern_AttachmentService_GetAttachment_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 1, 0, 4, 2, 5, 3}, []string{"api", "v1", "attachments", "name"}, ""))
pattern_AttachmentService_UpdateAttachment_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 1, 0, 4, 2, 5, 3}, []string{"api", "v1", "attachments", "attachment.name"}, ""))
@ -579,6 +644,7 @@ var (
var (
forward_AttachmentService_CreateAttachment_0 = runtime.ForwardResponseMessage
forward_AttachmentService_UploadAttachment_0 = runtime.ForwardResponseMessage
forward_AttachmentService_ListAttachments_0 = runtime.ForwardResponseMessage
forward_AttachmentService_GetAttachment_0 = runtime.ForwardResponseMessage
forward_AttachmentService_UpdateAttachment_0 = runtime.ForwardResponseMessage

View file

@ -21,6 +21,7 @@ const _ = grpc.SupportPackageIsVersion9
const (
AttachmentService_CreateAttachment_FullMethodName = "/memos.api.v1.AttachmentService/CreateAttachment"
AttachmentService_UploadAttachment_FullMethodName = "/memos.api.v1.AttachmentService/UploadAttachment"
AttachmentService_ListAttachments_FullMethodName = "/memos.api.v1.AttachmentService/ListAttachments"
AttachmentService_GetAttachment_FullMethodName = "/memos.api.v1.AttachmentService/GetAttachment"
AttachmentService_UpdateAttachment_FullMethodName = "/memos.api.v1.AttachmentService/UpdateAttachment"
@ -34,6 +35,11 @@ const (
type AttachmentServiceClient interface {
// CreateAttachment creates a new attachment.
CreateAttachment(ctx context.Context, in *CreateAttachmentRequest, opts ...grpc.CallOption) (*Attachment, error)
// UploadAttachment uploads a file in bounded chunks. The first call carries
// the spec and returns an upload_id; later calls carry that upload_id.
// Uploads are bound to the authenticated user, expire after 30 minutes of
// inactivity, and do not survive a server restart.
UploadAttachment(ctx context.Context, in *UploadAttachmentRequest, opts ...grpc.CallOption) (*UploadAttachmentResponse, error)
// ListAttachments lists all attachments.
ListAttachments(ctx context.Context, in *ListAttachmentsRequest, opts ...grpc.CallOption) (*ListAttachmentsResponse, error)
// GetAttachment returns an attachment by name.
@ -64,6 +70,16 @@ func (c *attachmentServiceClient) CreateAttachment(ctx context.Context, in *Crea
return out, nil
}
func (c *attachmentServiceClient) UploadAttachment(ctx context.Context, in *UploadAttachmentRequest, opts ...grpc.CallOption) (*UploadAttachmentResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(UploadAttachmentResponse)
err := c.cc.Invoke(ctx, AttachmentService_UploadAttachment_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *attachmentServiceClient) ListAttachments(ctx context.Context, in *ListAttachmentsRequest, opts ...grpc.CallOption) (*ListAttachmentsResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(ListAttachmentsResponse)
@ -120,6 +136,11 @@ func (c *attachmentServiceClient) BatchDeleteAttachments(ctx context.Context, in
type AttachmentServiceServer interface {
// CreateAttachment creates a new attachment.
CreateAttachment(context.Context, *CreateAttachmentRequest) (*Attachment, error)
// UploadAttachment uploads a file in bounded chunks. The first call carries
// the spec and returns an upload_id; later calls carry that upload_id.
// Uploads are bound to the authenticated user, expire after 30 minutes of
// inactivity, and do not survive a server restart.
UploadAttachment(context.Context, *UploadAttachmentRequest) (*UploadAttachmentResponse, error)
// ListAttachments lists all attachments.
ListAttachments(context.Context, *ListAttachmentsRequest) (*ListAttachmentsResponse, error)
// GetAttachment returns an attachment by name.
@ -143,6 +164,9 @@ type UnimplementedAttachmentServiceServer struct{}
func (UnimplementedAttachmentServiceServer) CreateAttachment(context.Context, *CreateAttachmentRequest) (*Attachment, error) {
return nil, status.Error(codes.Unimplemented, "method CreateAttachment not implemented")
}
func (UnimplementedAttachmentServiceServer) UploadAttachment(context.Context, *UploadAttachmentRequest) (*UploadAttachmentResponse, error) {
return nil, status.Error(codes.Unimplemented, "method UploadAttachment not implemented")
}
func (UnimplementedAttachmentServiceServer) ListAttachments(context.Context, *ListAttachmentsRequest) (*ListAttachmentsResponse, error) {
return nil, status.Error(codes.Unimplemented, "method ListAttachments not implemented")
}
@ -197,6 +221,24 @@ func _AttachmentService_CreateAttachment_Handler(srv interface{}, ctx context.Co
return interceptor(ctx, in, info, handler)
}
func _AttachmentService_UploadAttachment_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(UploadAttachmentRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(AttachmentServiceServer).UploadAttachment(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: AttachmentService_UploadAttachment_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(AttachmentServiceServer).UploadAttachment(ctx, req.(*UploadAttachmentRequest))
}
return interceptor(ctx, in, info, handler)
}
func _AttachmentService_ListAttachments_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ListAttachmentsRequest)
if err := dec(in); err != nil {
@ -298,6 +340,10 @@ var AttachmentService_ServiceDesc = grpc.ServiceDesc{
MethodName: "CreateAttachment",
Handler: _AttachmentService_CreateAttachment_Handler,
},
{
MethodName: "UploadAttachment",
Handler: _AttachmentService_UploadAttachment_Handler,
},
{
MethodName: "ListAttachments",
Handler: _AttachmentService_ListAttachments_Handler,

View file

@ -228,6 +228,35 @@ paths:
application/json:
schema:
$ref: '#/components/schemas/Status'
/api/v1/attachments:upload:
post:
tags:
- AttachmentService
description: |-
UploadAttachment uploads a file in bounded chunks. The first call carries
the spec and returns an upload_id; later calls carry that upload_id.
Uploads are bound to the authenticated user, expire after 30 minutes of
inactivity, and do not survive a server restart.
operationId: AttachmentService_UploadAttachment
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/UploadAttachmentRequest'
required: true
responses:
"200":
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/UploadAttachmentResponse'
default:
description: Default error response
content:
application/json:
schema:
$ref: '#/components/schemas/Status'
/api/v1/auth/me:
get:
tags:
@ -4573,6 +4602,76 @@ components:
contentType:
type: string
description: Optional. The MIME type of the input audio.
UploadAttachmentRequest:
required:
- writeOffset
type: object
properties:
spec:
allOf:
- $ref: '#/components/schemas/UploadAttachmentSpec'
description: Starts a new upload. The same call may also carry data and finish_write.
uploadId:
type: string
description: Continues the upload identified by a previous response.
writeOffset:
type: string
description: |-
Required. Zero-based byte offset at which data is written. Must equal the
committed size, except when retrying the most recently accepted chunk
with identical bytes and offset, which is accepted without writing again.
data:
type: string
description: |-
Optional. File bytes, at most max_chunk_size long. With no data and
finish_write false, the call reports progress without writing, and
write_offset is ignored.
format: bytes
finishWrite:
type: boolean
description: |-
Optional. Finalize the upload after writing data. The committed size must
then equal total_size. Any later call for the same upload_id returns the
created attachment.
UploadAttachmentResponse:
type: object
properties:
uploadId:
type: string
description: Opaque ID for subsequent calls. This is not a resource name.
committedSize:
type: string
description: Number of file bytes committed so far.
attachment:
allOf:
- $ref: '#/components/schemas/Attachment'
description: Set once the upload has been finalized.
maxChunkSize:
type: integer
description: Maximum number of data bytes accepted in one call.
format: int32
UploadAttachmentSpec:
required:
- attachment
type: object
properties:
attachment:
allOf:
- $ref: '#/components/schemas/Attachment'
description: |-
Required. Metadata for the attachment to create. content must be empty;
file bytes are sent in data.
attachmentId:
type: string
description: |-
Optional. The attachment ID to use for this attachment.
If empty, a unique ID will be generated.
Format: ^[a-zA-Z0-9]([a-zA-Z0-9-]{0,34}[a-zA-Z0-9])?$
totalSize:
type: string
description: |-
Optional. Total size of the file in bytes before media processing.
Zero represents an empty file.
UpsertMemoReactionRequest:
required:
- name

View file

@ -48,6 +48,7 @@ func TestPublicMethodsArePublic(t *testing.T) {
// TestProtectedMethodsRequireAuth verifies that non-public methods are recognized as protected.
func TestProtectedMethodsRequireAuth(t *testing.T) {
protectedMethods := []string{
"/memos.api.v1.AttachmentService/UploadAttachment",
// Auth Service - logout and get current user require auth
"/memos.api.v1.AuthService/SignOut",
"/memos.api.v1.AuthService/GetCurrentUser",

View file

@ -104,7 +104,7 @@ func TestStripImageExif(t *testing.T) {
t.Run("strip JPEG metadata", func(t *testing.T) {
t.Parallel()
strippedData, err := stripImageExif(originalData, "image/jpeg")
strippedData, err := stripImageExif(bytes.NewReader(originalData), "image/jpeg")
require.NoError(t, err)
assert.NotEmpty(t, strippedData)
@ -118,7 +118,7 @@ func TestStripImageExif(t *testing.T) {
t.Run("strip JPG metadata (alternate extension)", func(t *testing.T) {
t.Parallel()
strippedData, err := stripImageExif(originalData, "image/jpg")
strippedData, err := stripImageExif(bytes.NewReader(originalData), "image/jpg")
require.NoError(t, err)
assert.NotEmpty(t, strippedData)
@ -136,7 +136,7 @@ func TestStripImageExif(t *testing.T) {
err := imaging.Encode(&pngBuf, img, imaging.PNG)
require.NoError(t, err)
strippedData, err := stripImageExif(pngBuf.Bytes(), "image/png")
strippedData, err := stripImageExif(bytes.NewReader(pngBuf.Bytes()), "image/png")
require.NoError(t, err)
assert.NotEmpty(t, strippedData)
@ -151,7 +151,7 @@ func TestStripImageExif(t *testing.T) {
t.Parallel()
// WebP format will be converted to JPEG
strippedData, err := stripImageExif(originalData, "image/webp")
strippedData, err := stripImageExif(bytes.NewReader(originalData), "image/webp")
require.NoError(t, err)
assert.NotEmpty(t, strippedData)
@ -164,7 +164,7 @@ func TestStripImageExif(t *testing.T) {
t.Run("handle HEIC format by converting to JPEG", func(t *testing.T) {
t.Parallel()
strippedData, err := stripImageExif(originalData, "image/heic")
strippedData, err := stripImageExif(bytes.NewReader(originalData), "image/heic")
require.NoError(t, err)
assert.NotEmpty(t, strippedData)
@ -178,7 +178,7 @@ func TestStripImageExif(t *testing.T) {
t.Parallel()
invalidData := []byte("not an image")
_, err := stripImageExif(invalidData, "image/jpeg")
_, err := stripImageExif(bytes.NewReader(invalidData), "image/jpeg")
assert.Error(t, err)
assert.Contains(t, err.Error(), "failed to decode image")
})
@ -187,7 +187,7 @@ func TestStripImageExif(t *testing.T) {
t.Parallel()
emptyData := []byte{}
_, err := stripImageExif(emptyData, "image/jpeg")
_, err := stripImageExif(bytes.NewReader(emptyData), "image/jpeg")
assert.Error(t, err)
})
}
@ -195,7 +195,7 @@ func TestStripImageExif(t *testing.T) {
func TestValidateImagePixelCountRejectsOversizedDimensions(t *testing.T) {
t.Parallel()
err := validateImagePixelCount(testPNGHeaderWithDimensions(100_000, 100_000))
err := validateImageReaderPixelCount(bytes.NewReader(testPNGHeaderWithDimensions(100_000, 100_000)))
require.Error(t, err)
require.Contains(t, err.Error(), "image dimensions exceed maximum")
}
@ -203,7 +203,7 @@ func TestValidateImagePixelCountRejectsOversizedDimensions(t *testing.T) {
func TestStripImageExifRejectsOversizedDimensionsBeforeDecode(t *testing.T) {
t.Parallel()
_, err := stripImageExif(testPNGHeaderWithDimensions(100_000, 100_000), "image/png")
_, err := stripImageExif(bytes.NewReader(testPNGHeaderWithDimensions(100_000, 100_000)), "image/png")
require.Error(t, err)
require.Contains(t, err.Error(), "image dimensions exceed maximum")
}

View file

@ -1,10 +1,12 @@
package v1
import (
"bytes"
"context"
"encoding/binary"
"fmt"
"io"
"log/slog"
"math"
"mime"
"net/http"
"path/filepath"
@ -15,16 +17,16 @@ import (
"google.golang.org/grpc/status"
"google.golang.org/protobuf/types/known/emptypb"
"github.com/usememos/memos/internal/motionphoto"
v1pb "github.com/usememos/memos/proto/gen/api/v1"
storepb "github.com/usememos/memos/proto/gen/store"
"github.com/usememos/memos/store"
)
const (
// The upload memory buffer is 32 MiB.
// It should be kept low, so RAM usage doesn't get out of control.
// This is unrelated to maximum upload size limit, which is now set through system setting.
MaxUploadBufferSizeBytes = 32 << 20
MebiByte = 1024 * 1024
// DefaultUploadSizeLimitBytes applies when no upload size limit is configured.
DefaultUploadSizeLimitBytes = 32 << 20
MebiByte = 1024 * 1024
// defaultJPEGQuality is the JPEG quality used when re-encoding images for EXIF stripping.
// Quality 95 maintains visual quality while ensuring metadata is removed.
@ -71,7 +73,7 @@ func detectAttachmentMimeType(filename string, content []byte) string {
return http.DetectContentType(content)
}
func (s *APIV1Service) CreateAttachment(ctx context.Context, request *v1pb.CreateAttachmentRequest) (*v1pb.Attachment, error) {
func (s *APIV1Service) prepareAttachment(ctx context.Context, request *v1pb.CreateAttachmentRequest) (*store.Attachment, error) {
user, err := s.fetchCurrentUser(ctx)
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to get current user: %v", err)
@ -135,21 +137,6 @@ func (s *APIV1Service) CreateAttachment(ctx context.Context, request *v1pb.Creat
create.Payload.MediaMetadata = inputMediaMetadata
}
instanceStorageSetting, err := s.Store.GetInstanceStorageSetting(ctx)
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to get instance storage setting: %v", err)
}
size := binary.Size(request.Attachment.Content)
uploadSizeLimit := int(instanceStorageSetting.UploadSizeLimitMb) * MebiByte
if uploadSizeLimit == 0 {
uploadSizeLimit = MaxUploadBufferSizeBytes
}
if size > uploadSizeLimit {
return nil, status.Errorf(codes.InvalidArgument, "file size exceeds the limit")
}
create.Size = int64(size)
create.Blob = request.Attachment.Content
if request.Attachment.Memo != nil {
memoUID, err := ExtractMemoUIDFromName(*request.Attachment.Memo)
if err != nil {
@ -172,21 +159,77 @@ func (s *APIV1Service) CreateAttachment(ctx context.Context, request *v1pb.Creat
create.Policy = memoWritePolicy(user.ID, false)
}
if create.Payload == nil || create.Payload.MotionMedia == nil {
if detectedMotion := detectAndroidMotionMedia(create.Blob, create.Type, attachmentUID); detectedMotion != nil {
return create, nil
}
func (s *APIV1Service) CreateAttachment(ctx context.Context, request *v1pb.CreateAttachmentRequest) (*v1pb.Attachment, error) {
create, err := s.prepareAttachment(ctx, request)
if err != nil {
return nil, err
}
instanceStorageSetting, err := s.Store.GetInstanceStorageSetting(ctx)
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to get instance storage setting: %v", err)
}
content := request.Attachment.Content
if err := checkUploadSize(instanceStorageSetting, int64(len(content))); err != nil {
return nil, err
}
create.Size = int64(len(content))
return s.processAndSaveAttachment(ctx, create, instanceStorageSetting, bytes.NewReader(content))
}
func attachmentUploadLimit(setting *storepb.InstanceStorageSetting) int64 {
if setting.UploadSizeLimitMb <= 0 {
return DefaultUploadSizeLimitBytes
}
return min(setting.UploadSizeLimitMb, math.MaxInt64/MebiByte) * MebiByte
}
func checkUploadSize(setting *storepb.InstanceStorageSetting, size int64) error {
if size > attachmentUploadLimit(setting) {
return status.Errorf(codes.ResourceExhausted, "file size exceeds the limit")
}
return nil
}
// attachmentSource is the file content handed to the processing pipeline: a
// bytes.Reader for the one-shot RPC, an *os.File for a chunked upload.
type attachmentSource interface {
io.ReadSeeker
io.ReaderAt
}
// processAndSaveAttachment detects motion photos, strips EXIF metadata, stores
// the content, and creates the database row. create.Size must hold the source
// length on entry; it is updated when stripping re-encodes the image.
func (s *APIV1Service) processAndSaveAttachment(ctx context.Context, create *store.Attachment, instanceStorageSetting *storepb.InstanceStorageSetting, source attachmentSource) (*v1pb.Attachment, error) {
if create.Payload.GetMotionMedia() == nil && (create.Type == "image/jpeg" || create.Type == "image/jpg") {
detected, err := motionphoto.DetectJPEGReader(source, create.Size)
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to inspect motion photo: %v", err)
}
if detected != nil {
create.Payload = ensureAttachmentPayload(create.Payload)
create.Payload.MotionMedia = detectedMotion
create.Payload.MotionMedia = &storepb.MotionMedia{
Family: storepb.MotionMediaFamily_ANDROID_MOTION_PHOTO,
Role: storepb.MotionMediaRole_CONTAINER,
GroupId: create.UID,
PresentationTimestampUs: detected.PresentationTimestampUs,
HasEmbeddedVideo: true,
}
}
}
// Strip EXIF metadata from images for privacy protection.
// This removes sensitive information like GPS location, device details, etc.
content := io.ReadSeeker(source)
// Strip EXIF metadata from images for privacy protection. Motion photo
// containers are kept intact because re-encoding would drop the video.
if shouldStripExif(create.Type) && !isAndroidMotionContainer(create.Payload.GetMotionMedia()) {
release, err := s.acquireImageProcessingSlot(ctx)
if err != nil {
return nil, status.Errorf(codes.ResourceExhausted, "too many image processing requests")
}
strippedBlob, stripErr := stripImageExif(create.Blob, create.Type)
stripped, stripErr := stripImageExif(source, create.Type)
release()
if stripErr != nil {
// Log warning but continue with original image to ensure uploads don't fail.
@ -195,15 +238,21 @@ func (s *APIV1Service) CreateAttachment(ctx context.Context, request *v1pb.Creat
slog.String("filename", create.Filename),
slog.String("error", stripErr.Error()))
} else {
create.Blob = strippedBlob
create.Size = int64(len(strippedBlob))
content = bytes.NewReader(stripped)
create.Size = int64(len(stripped))
}
}
if err := saveAttachmentBlobWithInstanceStorageSetting(ctx, s.Profile, s.Store, create, instanceStorageSetting); err != nil {
if _, err := content.Seek(0, io.SeekStart); err != nil {
return nil, status.Errorf(codes.Internal, "failed to rewind attachment content: %v", err)
}
if err := saveAttachmentContent(ctx, s.Profile, s.Store, create, instanceStorageSetting, content); err != nil {
return nil, status.Errorf(codes.Internal, "failed to save attachment blob: %v", err)
}
return s.persistAttachment(ctx, create, instanceStorageSetting)
}
func (s *APIV1Service) persistAttachment(ctx context.Context, create *store.Attachment, instanceStorageSetting *storepb.InstanceStorageSetting) (*v1pb.Attachment, error) {
attachment, err := s.Store.CreateAttachment(ctx, create)
if err != nil {
createErr := mapMemoWriteError(err, "failed to create attachment")

View file

@ -4,13 +4,13 @@ import (
"bytes"
"context"
"image"
"io"
"github.com/disintegration/imaging"
"github.com/pkg/errors"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"github.com/usememos/memos/internal/motionphoto"
v1pb "github.com/usememos/memos/proto/gen/api/v1"
storepb "github.com/usememos/memos/proto/gen/store"
)
@ -38,25 +38,6 @@ func validateClientMotionMedia(motion *v1pb.MotionMedia, attachmentUID string) (
return storeMotion, nil
}
func detectAndroidMotionMedia(blob []byte, mimeType, attachmentUID string) *storepb.MotionMedia {
if mimeType != "image/jpeg" && mimeType != "image/jpg" {
return nil
}
detection := motionphoto.DetectJPEG(blob)
if detection == nil {
return nil
}
return &storepb.MotionMedia{
Family: storepb.MotionMediaFamily_ANDROID_MOTION_PHOTO,
Role: storepb.MotionMediaRole_CONTAINER,
GroupId: attachmentUID,
PresentationTimestampUs: detection.PresentationTimestampUs,
HasEmbeddedVideo: true,
}
}
// shouldStripExif checks if the MIME type is an image format that may contain EXIF metadata.
// Returns true for formats like JPEG, TIFF, WebP, HEIC, and HEIF which commonly contain
// privacy-sensitive metadata such as GPS coordinates, camera settings, and device information.
@ -76,8 +57,8 @@ func (s *APIV1Service) acquireImageProcessingSlot(ctx context.Context) (func(),
}, nil
}
func validateImagePixelCount(imageData []byte) error {
config, _, err := image.DecodeConfig(bytes.NewReader(imageData))
func validateImageReaderPixelCount(reader io.Reader) error {
config, _, err := image.DecodeConfig(reader)
if err != nil {
// Some formats supported by imaging do not expose dimensions through
// the standard image registry. Let the full decoder handle those.
@ -97,7 +78,8 @@ func validateImagePixelCount(imageData []byte) error {
//
// The function preserves the correct image orientation by applying EXIF orientation tags
// during decoding before stripping all metadata. Images are re-encoded with high quality
// to minimize visual degradation.
// to minimize visual degradation. The re-encoded output is returned in memory; its size
// is bounded by maxImagePixels, which the decoder already has to hold.
//
// Supported formats:
// - JPEG/JPG: Re-encoded as JPEG with quality 95
@ -105,34 +87,28 @@ func validateImagePixelCount(imageData []byte) error {
// - TIFF/WebP/HEIC/HEIF: Re-encoded as JPEG with quality 95
//
// Returns the cleaned image data without any EXIF metadata, or an error if processing fails.
func stripImageExif(imageData []byte, mimeType string) ([]byte, error) {
if err := validateImagePixelCount(imageData); err != nil {
func stripImageExif(source io.ReadSeeker, mimeType string) ([]byte, error) {
if _, err := source.Seek(0, io.SeekStart); err != nil {
return nil, errors.Wrap(err, "failed to rewind image")
}
if err := validateImageReaderPixelCount(source); err != nil {
return nil, err
}
// Decode image with automatic EXIF orientation correction.
// This ensures the image displays correctly after metadata removal.
img, err := imaging.Decode(bytes.NewReader(imageData), imaging.AutoOrientation(true))
if _, err := source.Seek(0, io.SeekStart); err != nil {
return nil, errors.Wrap(err, "failed to rewind image")
}
img, err := imaging.Decode(source, imaging.AutoOrientation(true))
if err != nil {
return nil, errors.Wrap(err, "failed to decode image")
}
// Re-encode the image without EXIF metadata.
var buf bytes.Buffer
var encodeErr error
if mimeType == "image/png" {
// Preserve PNG format for lossless encoding
encodeErr = imaging.Encode(&buf, img, imaging.PNG)
err = imaging.Encode(&buf, img, imaging.PNG)
} else {
// For JPEG, TIFF, WebP, HEIC, HEIF - re-encode as JPEG.
// This ensures EXIF is stripped and provides good compression.
encodeErr = imaging.Encode(&buf, img, imaging.JPEG, imaging.JPEGQuality(defaultJPEGQuality))
err = imaging.Encode(&buf, img, imaging.JPEG, imaging.JPEGQuality(defaultJPEGQuality))
}
if encodeErr != nil {
return nil, errors.Wrap(encodeErr, "failed to encode image")
if err != nil {
return nil, errors.Wrap(err, "failed to encode image")
}
return buf.Bytes(), nil
}

View file

@ -1,7 +1,6 @@
package v1
import (
"bytes"
"context"
"fmt"
"io"
@ -46,21 +45,29 @@ func convertAttachmentFromStore(attachment *store.Attachment) *v1pb.Attachment {
return attachmentMessage
}
// SaveAttachmentBlob saves the blob of attachment based on the storage config.
func SaveAttachmentBlob(ctx context.Context, profile *profile.Profile, stores *store.Store, create *store.Attachment) error {
instanceStorageSetting, err := stores.GetInstanceStorageSetting(ctx)
if err != nil {
return errors.Wrap(err, "Failed to find instance storage setting")
}
return saveAttachmentBlobWithInstanceStorageSetting(ctx, profile, stores, create, instanceStorageSetting)
// attachmentContextReader stops a long local copy once the request context is
// canceled; io.Copy has no context of its own, unlike the S3 client.
type attachmentContextReader struct {
ctx context.Context
reader io.Reader
}
func saveAttachmentBlobWithInstanceStorageSetting(
func (r *attachmentContextReader) Read(p []byte) (int, error) {
if err := r.ctx.Err(); err != nil {
return 0, err
}
return r.reader.Read(p)
}
// saveAttachmentContent writes content to the default storage and records
// where it went on create. Database storage keeps the bytes on create.Blob.
func saveAttachmentContent(
ctx context.Context,
profile *profile.Profile,
stores *store.Store,
create *store.Attachment,
instanceStorageSetting *storepb.InstanceStorageSetting,
content io.Reader,
) error {
defaultStorage := store.GetDefaultStorage(instanceStorageSetting)
if defaultStorage == nil {
@ -99,9 +106,24 @@ func saveAttachmentBlobWithInstanceStorageSetting(
return errors.Wrap(err, "Failed to create directory")
}
// Write the blob to the file.
if err := os.WriteFile(osPath, create.Blob, 0644); err != nil {
return errors.Wrap(err, "Failed to write file")
// Stage in a temp file so partial content never appears at the final path.
file, err := os.CreateTemp(dir, ".memos-upload-*")
if err != nil {
return errors.Wrap(err, "failed to create attachment file")
}
defer os.Remove(file.Name())
defer file.Close()
if _, err := io.Copy(file, &attachmentContextReader{ctx: ctx, reader: content}); err != nil {
return errors.Wrap(err, "failed to write attachment file")
}
if err := file.Chmod(0644); err != nil {
return errors.Wrap(err, "failed to set attachment permissions")
}
if err := file.Close(); err != nil {
return errors.Wrap(err, "failed to close attachment file")
}
if err := os.Rename(file.Name(), osPath); err != nil {
return errors.Wrap(err, "failed to finalize attachment file")
}
create.Reference = internalPath
create.Blob = nil
@ -117,7 +139,7 @@ func saveAttachmentBlobWithInstanceStorageSetting(
filepathTemplate = filepath.Join(filepathTemplate, "{filename}")
}
filepathTemplate = replaceFilenameWithPathTemplate(filepathTemplate, create.Filename)
key, err := driver.UploadObject(ctx, filepathTemplate, create.Type, bytes.NewReader(create.Blob))
key, err := driver.UploadObject(ctx, filepathTemplate, create.Type, content)
if err != nil {
return errors.Wrap(err, "failed to upload via storage driver")
}
@ -133,6 +155,12 @@ func saveAttachmentBlobWithInstanceStorageSetting(
},
}
create.Payload = payload
} else {
blob, err := io.ReadAll(content)
if err != nil {
return errors.Wrap(err, "failed to read attachment content")
}
create.Blob = blob
}
return nil

View file

@ -0,0 +1,214 @@
package v1
import (
"context"
"crypto/sha256"
"io"
"os"
"time"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/proto"
v1pb "github.com/usememos/memos/proto/gen/api/v1"
"github.com/usememos/memos/store"
)
// UploadAttachment accepts bounded unary chunks. A call carrying a spec opens
// a new upload; a call carrying an upload ID continues one. Either kind may
// write data and finalize.
func (s *APIV1Service) UploadAttachment(ctx context.Context, request *v1pb.UploadAttachmentRequest) (*v1pb.UploadAttachmentResponse, error) {
user, err := s.requireCurrentSpaceUser(ctx)
if err != nil {
return nil, err
}
if len(request.Data) > attachmentUploadChunkSize {
return nil, status.Errorf(codes.ResourceExhausted, "upload chunk exceeds the limit")
}
var id string
var upload *attachmentUpload
switch u := request.Upload.(type) {
case *v1pb.UploadAttachmentRequest_Spec:
id, upload, err = s.startAttachmentUpload(ctx, request, u.Spec, user.ID)
if err != nil {
return nil, err
}
upload.mu.Lock()
case *v1pb.UploadAttachmentRequest_UploadId:
id = u.UploadId
upload, err = s.attachmentUploads.get(id, user.ID)
if err != nil {
return nil, err
}
upload.mu.Lock()
if !time.Now().Before(upload.expireTime) {
upload.mu.Unlock()
return nil, status.Errorf(codes.NotFound, "upload not found or expired")
}
default:
return nil, status.Errorf(codes.InvalidArgument, "spec or upload_id is required")
}
defer upload.mu.Unlock()
if err := ctx.Err(); err != nil {
return nil, status.FromContextError(err).Err()
}
if err := upload.write(ctx, request); err != nil {
return nil, err
}
var attachment *v1pb.Attachment
if upload.complete {
// Recheck access and existence; a completed upload must not resurrect a
// deleted attachment or disclose metadata after access is revoked.
attachment, err = s.GetAttachment(ctx, &v1pb.GetAttachmentRequest{Name: "attachments/" + upload.uid})
} else if request.FinishWrite {
if upload.committedSize != upload.totalSize {
return nil, status.Errorf(codes.FailedPrecondition, "upload is incomplete")
}
attachment, err = s.finishAttachmentUpload(ctx, upload)
if err == nil {
upload.complete = true
os.Remove(upload.path)
}
}
if err != nil {
return nil, err
}
upload.expireTime = time.Now().Add(attachmentUploadTTL)
return &v1pb.UploadAttachmentResponse{
UploadId: id, CommittedSize: upload.committedSize, Attachment: attachment, MaxChunkSize: attachmentUploadChunkSize,
}, nil
}
// startAttachmentUpload validates the spec and registers a new upload. Every
// check that can fail on the accompanying data runs first, so a rejected call
// never leaves an orphaned upload behind.
func (s *APIV1Service) startAttachmentUpload(ctx context.Context, request *v1pb.UploadAttachmentRequest, spec *v1pb.UploadAttachmentSpec, ownerID int32) (string, *attachmentUpload, error) {
if spec.Attachment == nil {
return "", nil, status.Errorf(codes.InvalidArgument, "spec.attachment is required")
}
if len(spec.Attachment.Content) != 0 {
return "", nil, status.Errorf(codes.InvalidArgument, "spec.attachment.content must be empty; send file bytes in data")
}
if proto.Size(spec) > attachmentUploadMetadataLimit || spec.TotalSize < 0 {
return "", nil, status.Errorf(codes.InvalidArgument, "invalid attachment metadata or total_size")
}
if request.WriteOffset != 0 {
return "", nil, status.Errorf(codes.OutOfRange, "write_offset must be 0 for a new upload")
}
if int64(len(request.Data)) > spec.TotalSize {
return "", nil, status.Errorf(codes.InvalidArgument, "data exceeds total_size")
}
setting, err := s.Store.GetInstanceStorageSetting(ctx)
if err != nil {
return "", nil, status.Errorf(codes.Internal, "failed to get storage setting: %v", err)
}
if err := checkUploadSize(setting, spec.TotalSize); err != nil {
return "", nil, err
}
// Validate before allocating a temporary file. Keep the original MIME type
// so an omitted type can be sniffed from real bytes at finalization.
metadata := proto.CloneOf(spec.Attachment)
validationMetadata := proto.CloneOf(metadata)
if validationMetadata.Type == "" {
// Type-dependent metadata validation must wait for content sniffing.
validationMetadata.MediaMetadata = nil
}
create, err := s.prepareAttachment(ctx, &v1pb.CreateAttachmentRequest{Attachment: validationMetadata, AttachmentId: spec.AttachmentId})
if err != nil {
return "", nil, err
}
upload := &attachmentUpload{ownerID: ownerID, metadata: metadata, uid: create.UID, totalSize: spec.TotalSize}
id, err := s.attachmentUploads.create(s.Profile.Data, upload)
if err != nil {
return "", nil, err
}
return id, upload, nil
}
func (u *attachmentUpload) write(ctx context.Context, request *v1pb.UploadAttachmentRequest) error {
if len(request.Data) == 0 {
if request.FinishWrite && request.WriteOffset != u.committedSize {
return status.Errorf(codes.OutOfRange, "write_offset must equal committed_size")
}
return nil
}
digest := sha256.Sum256(request.Data)
if u.committedSize > 0 && request.WriteOffset+int64(len(request.Data)) == u.committedSize && digest == u.lastDigest {
return nil // A lost response can be retried without appending bytes twice.
}
if u.complete {
return status.Errorf(codes.FailedPrecondition, "upload is already complete")
}
if request.WriteOffset != u.committedSize {
return status.Errorf(codes.OutOfRange, "write_offset must equal committed_size")
}
if int64(len(request.Data)) > u.totalSize-u.committedSize {
return status.Errorf(codes.InvalidArgument, "data exceeds total_size")
}
file, err := os.OpenFile(u.path, os.O_WRONLY, 0600)
if err != nil {
return status.Errorf(codes.Internal, "failed to open upload file: %v", err)
}
defer file.Close()
n, err := file.WriteAt(request.Data, u.committedSize)
if err == nil && n != len(request.Data) {
err = io.ErrShortWrite
}
if err == nil {
err = ctx.Err()
}
if err != nil {
if truncateErr := file.Truncate(u.committedSize); truncateErr != nil {
// The file no longer matches committedSize; revoke the upload.
u.expireTime = time.Time{}
os.Remove(u.path)
}
return status.Errorf(codes.Internal, "failed to write upload chunk: %v", err)
}
u.lastDigest = digest
u.committedSize += int64(n)
return nil
}
func (s *APIV1Service) finishAttachmentUpload(ctx context.Context, upload *attachmentUpload) (*v1pb.Attachment, error) {
if upload.finalizeAttempted {
// A database operation may commit before returning an error. Resolve that
// outcome before saving another object under the upload's stable UID.
persisted, err := s.Store.GetAttachment(ctx, &store.FindAttachment{UID: &upload.uid})
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to resolve previous finalization: %v", err)
}
if persisted != nil {
if persisted.CreatorID != upload.ownerID {
return nil, status.Errorf(codes.AlreadyExists, "attachment ID already exists")
}
return s.GetAttachment(ctx, &v1pb.GetAttachmentRequest{Name: "attachments/" + upload.uid})
}
}
setting, err := s.Store.GetInstanceStorageSetting(ctx)
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to get storage setting: %v", err)
}
if err := checkUploadSize(setting, upload.totalSize); err != nil {
return nil, err
}
file, err := os.Open(upload.path)
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to open upload file: %v", err)
}
defer file.Close()
metadata := proto.CloneOf(upload.metadata)
metadata.Content = make([]byte, min(upload.totalSize, 512))
if _, err := io.ReadFull(file, metadata.Content); err != nil {
return nil, status.Errorf(codes.Internal, "failed to read attachment header: %v", err)
}
// Revalidate memo permissions and media metadata with the actual MIME type.
create, err := s.prepareAttachment(ctx, &v1pb.CreateAttachmentRequest{Attachment: metadata, AttachmentId: upload.uid})
if err != nil {
return nil, err
}
create.Size = upload.totalSize
upload.finalizeAttempted = true
return s.processAndSaveAttachment(ctx, create, setting, file)
}

View file

@ -0,0 +1,194 @@
package v1
import (
"crypto/rand"
"crypto/sha256"
"os"
"path/filepath"
"sync"
"time"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
v1pb "github.com/usememos/memos/proto/gen/api/v1"
)
const (
attachmentUploadChunkSize = 2 << 20
attachmentUploadRequestLimit = 4 << 20
attachmentUploadMetadataLimit = 64 << 10
attachmentUploadTTL = 30 * time.Minute
attachmentUploadMaxSessions = 1024
attachmentUploadMaxActive = 128
attachmentUploadMaxActivePerUser = 8
attachmentUploadTempPrefix = ".memos-rpc-upload-"
attachmentUploadProcedure = "/memos.api.v1.AttachmentService/UploadAttachment"
)
// Upload state is private to the process; only Attachment is an API resource.
// Each upload holds metadata and the last chunk's digest, never its contents.
type attachmentUpload struct {
mu sync.Mutex
ownerID int32
metadata *v1pb.Attachment
uid string
path string
totalSize int64
committedSize int64
// expireTime is zeroed to revoke an upload before its TTL elapses.
expireTime time.Time
// lastDigest identifies the most recently accepted chunk so a client can
// safely resend it after losing the response.
lastDigest [sha256.Size]byte
finalizeAttempted bool
complete bool
}
type attachmentUploads struct {
mu sync.Mutex
entries map[string]*attachmentUpload
stop chan struct{}
done chan struct{}
closed bool
}
func (m *attachmentUploads) startLocked(dir string) {
if m.entries != nil {
return
}
m.entries = make(map[string]*attachmentUpload)
m.stop = make(chan struct{})
m.done = make(chan struct{})
go func() {
defer close(m.done)
ticker := time.NewTicker(time.Minute)
defer ticker.Stop()
for {
select {
case <-m.stop:
return
case now := <-ticker.C:
m.mu.Lock()
m.sweepLocked(now, 0)
m.removeOrphansLocked(dir, now)
m.mu.Unlock()
}
}
}()
}
// sweepLocked drops expired uploads and counts the unfinished ones, both in
// total and for ownerID. An upload locked by an in-flight request is
// unfinished by definition.
func (m *attachmentUploads) sweepLocked(now time.Time, ownerID int32) (owned, total int) {
count := func(upload *attachmentUpload) {
total++
if upload.ownerID == ownerID {
owned++
}
}
for id, upload := range m.entries {
// Never hold the manager lock while waiting for a file write or finalize.
if !upload.mu.TryLock() {
count(upload)
continue
}
if !now.Before(upload.expireTime) {
os.Remove(upload.path)
delete(m.entries, id)
} else if !upload.complete {
count(upload)
}
upload.mu.Unlock()
}
return owned, total
}
func (m *attachmentUploads) removeOrphansLocked(dir string, now time.Time) {
tracked := make(map[string]bool, len(m.entries))
for _, upload := range m.entries {
tracked[upload.path] = true // path is immutable after insertion.
}
paths, _ := filepath.Glob(filepath.Join(dir, attachmentUploadTempPrefix+"*"))
for _, path := range paths {
if tracked[path] {
continue
}
if info, err := os.Stat(path); err == nil && info.Mode().IsRegular() && now.Sub(info.ModTime()) >= attachmentUploadTTL {
os.Remove(path)
}
}
}
func (m *attachmentUploads) create(dir string, upload *attachmentUpload) (string, error) {
m.mu.Lock()
defer m.mu.Unlock()
if m.closed {
return "", status.Errorf(codes.Unavailable, "server is shutting down")
}
m.startLocked(dir)
owned, total := m.sweepLocked(time.Now(), upload.ownerID)
if owned >= attachmentUploadMaxActivePerUser {
return "", status.Errorf(codes.ResourceExhausted, "too many active uploads")
}
if len(m.entries) >= attachmentUploadMaxSessions || total >= attachmentUploadMaxActive {
return "", status.Errorf(codes.ResourceExhausted, "too many uploads")
}
file, err := os.CreateTemp(dir, attachmentUploadTempPrefix+"*")
if err != nil {
return "", status.Errorf(codes.Internal, "failed to create upload file: %v", err)
}
if err := file.Close(); err != nil {
os.Remove(file.Name())
return "", status.Errorf(codes.Internal, "failed to close upload file: %v", err)
}
upload.path = file.Name()
upload.expireTime = time.Now().Add(attachmentUploadTTL)
id := rand.Text()
m.entries[id] = upload
return id, nil
}
func (m *attachmentUploads) get(id string, ownerID int32) (*attachmentUpload, error) {
m.mu.Lock()
upload := m.entries[id]
closed := m.closed
m.mu.Unlock()
if closed {
return nil, status.Errorf(codes.Unavailable, "server is shutting down")
}
if upload == nil || upload.ownerID != ownerID {
return nil, status.Errorf(codes.NotFound, "upload not found or expired")
}
return upload, nil
}
// CloseAttachmentUploads stops expiration work and removes pending upload files.
// Call it after draining HTTP requests during server shutdown.
func (s *APIV1Service) CloseAttachmentUploads() {
m := &s.attachmentUploads
m.mu.Lock()
if m.closed {
m.mu.Unlock()
return
}
m.closed = true
if m.stop != nil {
close(m.stop)
}
done := m.done
m.mu.Unlock()
if done != nil {
<-done
}
m.mu.Lock()
defer m.mu.Unlock()
for id, upload := range m.entries {
upload.mu.Lock()
upload.expireTime = time.Time{}
os.Remove(upload.path)
upload.mu.Unlock()
delete(m.entries, id)
}
}

View file

@ -0,0 +1,357 @@
package v1
import (
"bytes"
"context"
"io"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"github.com/stretchr/testify/require"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/proto"
"github.com/usememos/memos/internal/testutil"
v1pb "github.com/usememos/memos/proto/gen/api/v1"
storepb "github.com/usememos/memos/proto/gen/store"
"github.com/usememos/memos/store"
)
func newUploadTestService(t *testing.T) (*APIV1Service, context.Context) {
t.Helper()
svc := newIntegrationService(t)
t.Cleanup(svc.CloseAttachmentUploads)
user := createSpaceTestUser(context.Background(), t, svc, "uploader", store.RoleUser)
return svc, userCtx(context.Background(), user.ID)
}
func uploadSpec(filename string, size int64) *v1pb.UploadAttachmentRequest_Spec {
return &v1pb.UploadAttachmentRequest_Spec{Spec: &v1pb.UploadAttachmentSpec{Attachment: &v1pb.Attachment{Filename: filename}, TotalSize: size}}
}
func uploadID(id string) *v1pb.UploadAttachmentRequest_UploadId {
return &v1pb.UploadAttachmentRequest_UploadId{UploadId: id}
}
func startTestUpload(ctx context.Context, t *testing.T, svc *APIV1Service, size int64) string {
t.Helper()
spec := uploadSpec("file.bin", size)
spec.Spec.Attachment.Type = "application/octet-stream"
response, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: spec})
require.NoError(t, err)
require.NotEmpty(t, response.UploadId)
require.EqualValues(t, attachmentUploadChunkSize, response.MaxChunkSize)
require.Zero(t, response.CommittedSize)
require.Nil(t, response.Attachment)
return response.UploadId
}
func TestUploadAttachmentChunksAndRetries(t *testing.T) {
svc, ctx := newUploadTestService(t)
id := startTestUpload(ctx, t, svc, 6)
first := &v1pb.UploadAttachmentRequest{Upload: uploadID(id), Data: []byte("abc")}
for range 2 {
response, err := svc.UploadAttachment(ctx, first)
require.NoError(t, err)
require.Equal(t, id, response.UploadId)
require.EqualValues(t, 3, response.CommittedSize)
}
for _, request := range []*v1pb.UploadAttachmentRequest{
{Upload: uploadID(id), Data: []byte("xyz")},
{Upload: uploadID(id), WriteOffset: 4, Data: []byte("d")},
{Upload: uploadID(id), WriteOffset: -1, Data: []byte("d")},
} {
_, err := svc.UploadAttachment(ctx, request)
require.Equal(t, codes.OutOfRange, status.Code(err))
}
progress, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(id), WriteOffset: -1})
require.NoError(t, err)
require.EqualValues(t, 3, progress.CommittedSize)
_, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(id), WriteOffset: 3, FinishWrite: true})
require.Equal(t, codes.FailedPrecondition, status.Code(err))
last := &v1pb.UploadAttachmentRequest{Upload: uploadID(id), WriteOffset: 3, Data: []byte("def"), FinishWrite: true}
result, err := svc.UploadAttachment(ctx, last)
require.NoError(t, err)
require.EqualValues(t, 6, result.Attachment.Size)
for _, request := range []*v1pb.UploadAttachmentRequest{last, {Upload: uploadID(id)}, {Upload: uploadID(id), WriteOffset: 6, FinishWrite: true}} {
repeated, err := svc.UploadAttachment(ctx, request)
require.NoError(t, err)
require.Equal(t, result.Attachment.Name, repeated.Attachment.Name)
}
rows, err := svc.Store.ListAttachments(ctx, &store.FindAttachment{})
require.NoError(t, err)
require.Len(t, rows, 1)
content, err := os.ReadFile(rows[0].Reference)
require.NoError(t, err)
require.Equal(t, "abcdef", string(content))
require.NoFileExists(t, svc.attachmentUploads.entries[id].path)
_, err = svc.DeleteAttachment(ctx, &v1pb.DeleteAttachmentRequest{Name: result.Attachment.Name})
require.NoError(t, err)
_, err = svc.UploadAttachment(ctx, last)
require.Equal(t, codes.NotFound, status.Code(err))
}
func TestUploadAttachmentSingleCall(t *testing.T) {
svc, ctx := newUploadTestService(t)
spec := uploadSpec("note.txt", 3)
spec.Spec.AttachmentId = "chosen-id"
result, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: spec, Data: []byte("abc"), FinishWrite: true})
require.NoError(t, err)
require.Equal(t, "attachments/chosen-id", result.Attachment.Name)
require.EqualValues(t, 3, result.Attachment.Size)
require.EqualValues(t, 3, result.CommittedSize)
empty, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadSpec("empty.txt", 0), FinishWrite: true})
require.NoError(t, err)
require.Zero(t, empty.Attachment.Size)
// A spec call may carry the first chunk without finishing.
partial, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadSpec("two.bin", 4), Data: []byte("ab")})
require.NoError(t, err)
require.Nil(t, partial.Attachment)
require.EqualValues(t, 2, partial.CommittedSize)
done, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(partial.UploadId), WriteOffset: 2, Data: []byte("cd"), FinishWrite: true})
require.NoError(t, err)
require.EqualValues(t, 4, done.Attachment.Size)
rows, err := svc.Store.ListAttachments(ctx, &store.FindAttachment{})
require.NoError(t, err)
require.Len(t, rows, 3)
}
func TestUploadAttachmentValidation(t *testing.T) {
svc, ctx := newUploadTestService(t)
_, err := svc.UploadAttachment(context.Background(), &v1pb.UploadAttachmentRequest{})
require.Equal(t, codes.Unauthenticated, status.Code(err))
badID := uploadSpec("file", 0)
badID.Spec.AttachmentId = "bad id!"
withContent := uploadSpec("file", 0)
withContent.Spec.Attachment.Content = []byte("not metadata")
for _, request := range []*v1pb.UploadAttachmentRequest{
{},
{Upload: &v1pb.UploadAttachmentRequest_Spec{Spec: &v1pb.UploadAttachmentSpec{}}},
{Upload: uploadSpec("file", -1)},
{Upload: uploadSpec("../file", 0)},
{Upload: withContent},
{Upload: badID},
{Upload: uploadSpec(strings.Repeat("x", attachmentUploadMetadataLimit+1), 0)},
{Upload: uploadSpec("file", 3), Data: []byte("four")},
} {
_, err := svc.UploadAttachment(ctx, request)
require.Equal(t, codes.InvalidArgument, status.Code(err), request.String())
}
_, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadSpec("file", 3), WriteOffset: 1, Data: []byte("abc")})
require.Equal(t, codes.OutOfRange, status.Code(err))
_, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadSpec("file", 1<<60)})
require.Equal(t, codes.ResourceExhausted, status.Code(err))
require.Empty(t, svc.attachmentUploads.entries, "rejected spec calls must not register uploads")
id := startTestUpload(ctx, t, svc, 3)
other := createSpaceTestUser(context.Background(), t, svc, "other-uploader", store.RoleUser)
_, err = svc.UploadAttachment(userCtx(context.Background(), other.ID), &v1pb.UploadAttachmentRequest{Upload: uploadID(id)})
require.Equal(t, codes.NotFound, status.Code(err))
_, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID("unknown")})
require.Equal(t, codes.NotFound, status.Code(err))
_, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(id), Data: make([]byte, attachmentUploadChunkSize+1)})
require.Equal(t, codes.ResourceExhausted, status.Code(err))
_, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(id), Data: []byte("four")})
require.Equal(t, codes.InvalidArgument, status.Code(err))
canceled, cancel := context.WithCancel(ctx)
cancel()
_, err = svc.UploadAttachment(canceled, &v1pb.UploadAttachmentRequest{Upload: uploadID(id), Data: []byte("abc")})
require.Error(t, err)
progress, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(id)})
require.NoError(t, err)
require.Zero(t, progress.CommittedSize)
}
func TestUploadAttachmentEmptyAndConcurrentFinish(t *testing.T) {
svc, ctx := newUploadTestService(t)
id := startTestUpload(ctx, t, svc, 0)
var wg sync.WaitGroup
results := make(chan *v1pb.UploadAttachmentResponse, 8)
errs := make(chan error, 8)
for range 8 {
wg.Go(func() {
response, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(id), FinishWrite: true})
results <- response
errs <- err
})
}
wg.Wait()
close(results)
close(errs)
for err := range errs {
require.NoError(t, err)
}
name := ""
for response := range results {
if name == "" {
name = response.Attachment.Name
}
require.Equal(t, name, response.Attachment.Name)
require.Zero(t, response.Attachment.Size)
}
rows, err := svc.Store.ListAttachments(ctx, &store.FindAttachment{})
require.NoError(t, err)
require.Len(t, rows, 1)
}
func TestUploadAttachmentFinalizationRecovery(t *testing.T) {
svc, ctx := newUploadTestService(t)
id := startTestUpload(ctx, t, svc, 3)
last := &v1pb.UploadAttachmentRequest{Upload: uploadID(id), Data: []byte("abc"), FinishWrite: true}
_, err := svc.UploadAttachment(store.WithCreateAttachmentPostCommitFailpoint(ctx), last)
require.Equal(t, codes.Internal, status.Code(err))
result, err := svc.UploadAttachment(ctx, last)
require.NoError(t, err)
require.NotNil(t, result.Attachment)
rows, err := svc.Store.ListAttachments(ctx, &store.FindAttachment{})
require.NoError(t, err)
require.Len(t, rows, 1)
content, err := os.ReadFile(rows[0].Reference)
require.NoError(t, err)
require.Equal(t, "abc", string(content))
}
func TestUploadAttachmentRechecksMemoAndSize(t *testing.T) {
svc, ctx := newUploadTestService(t)
memo, err := svc.CreateMemo(ctx, &v1pb.CreateMemoRequest{Memo: &v1pb.Memo{Content: "upload target"}})
require.NoError(t, err)
spec := uploadSpec("file.txt", 3)
spec.Spec.Attachment.Memo = &memo.Name
initial, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: spec})
require.NoError(t, err)
_, err = svc.DeleteMemo(ctx, &v1pb.DeleteMemoRequest{Name: memo.Name, Force: true})
require.NoError(t, err)
_, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(initial.UploadId), Data: []byte("abc"), FinishWrite: true})
require.Equal(t, codes.NotFound, status.Code(err))
id := startTestUpload(ctx, t, svc, 2*MebiByte)
_, err = svc.Store.UpsertInstanceSetting(ctx, &storepb.InstanceSetting{Key: storepb.InstanceSettingKey_STORAGE,
Value: &storepb.InstanceSetting_StorageSetting{StorageSetting: &storepb.InstanceStorageSetting{UploadSizeLimitMb: 1}},
})
require.NoError(t, err)
_, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(id), Data: make([]byte, 2*MebiByte), FinishWrite: true})
require.Equal(t, codes.ResourceExhausted, status.Code(err))
rows, err := svc.Store.ListAttachments(ctx, &store.FindAttachment{})
require.NoError(t, err)
require.Empty(t, rows)
}
func TestUploadAttachmentExpiryAndLimits(t *testing.T) {
svc, ctx := newUploadTestService(t)
ids := make([]string, attachmentUploadMaxActivePerUser)
for i := range ids {
ids[i] = startTestUpload(ctx, t, svc, 0)
}
_, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadSpec("too-many", 0)})
require.Equal(t, codes.ResourceExhausted, status.Code(err))
_, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(ids[0]), FinishWrite: true})
require.NoError(t, err)
startTestUpload(ctx, t, svc, 0) // Completed uploads do not consume an active slot.
upload := svc.attachmentUploads.entries[ids[1]]
upload.mu.Lock()
upload.expireTime = time.Now().Add(-time.Second)
upload.mu.Unlock()
_, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(ids[1])})
require.Equal(t, codes.NotFound, status.Code(err))
orphan := filepath.Join(svc.Profile.Data, attachmentUploadTempPrefix+"orphan")
recent := filepath.Join(svc.Profile.Data, attachmentUploadTempPrefix+"recent")
unrelated := filepath.Join(svc.Profile.Data, "keep.txt")
for _, path := range []string{orphan, recent, unrelated} {
require.NoError(t, os.WriteFile(path, []byte("x"), 0600))
}
old := time.Now().Add(-2 * attachmentUploadTTL)
require.NoError(t, os.Chtimes(orphan, old, old))
require.NoError(t, os.Chtimes(unrelated, old, old))
svc.attachmentUploads.mu.Lock()
svc.attachmentUploads.sweepLocked(time.Now(), 0)
svc.attachmentUploads.removeOrphansLocked(svc.Profile.Data, time.Now())
svc.attachmentUploads.mu.Unlock()
require.NoFileExists(t, upload.path)
require.NoFileExists(t, orphan)
require.FileExists(t, recent)
require.FileExists(t, unrelated)
svc.CloseAttachmentUploads()
require.Empty(t, svc.attachmentUploads.entries)
}
func TestUploadAttachmentAboveLegacyRequestLimit(t *testing.T) {
svc, ctx := newUploadTestService(t)
const size = 300 * MebiByte
_, err := svc.Store.UpsertInstanceSetting(ctx, &storepb.InstanceSetting{Key: storepb.InstanceSettingKey_STORAGE,
Value: &storepb.InstanceSetting_StorageSetting{StorageSetting: &storepb.InstanceStorageSetting{UploadSizeLimitMb: 400}},
})
require.NoError(t, err)
id := startTestUpload(ctx, t, svc, size)
chunk := bytes.Repeat([]byte{0x5a}, attachmentUploadChunkSize)
var response *v1pb.UploadAttachmentResponse
for offset := int64(0); offset < size; offset += int64(len(chunk)) {
response, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{
Upload: uploadID(id), WriteOffset: offset, Data: chunk, FinishWrite: offset+int64(len(chunk)) == size,
})
require.NoError(t, err)
}
require.EqualValues(t, size, response.Attachment.Size)
rows, err := svc.Store.ListAttachments(ctx, &store.FindAttachment{})
require.NoError(t, err)
require.Len(t, rows, 1)
file, err := os.Open(rows[0].Reference)
require.NoError(t, err)
defer file.Close()
info, err := file.Stat()
require.NoError(t, err)
require.EqualValues(t, size, info.Size())
_, err = file.Seek(-int64(len(chunk)), io.SeekEnd)
require.NoError(t, err)
tail := make([]byte, len(chunk))
_, err = io.ReadFull(file, tail)
require.NoError(t, err)
require.Equal(t, chunk, tail)
}
func TestUploadAttachmentMediaProcessing(t *testing.T) {
for _, tc := range []struct {
name string
content []byte
motion bool
}{
{name: "JPEG", content: testutil.BuildJPEG(20, 10)},
{name: "motion photo", content: testutil.BuildMotionPhotoJPEG(), motion: true},
} {
t.Run(tc.name, func(t *testing.T) {
svc, ctx := newUploadTestService(t)
metadata := &v1pb.Attachment{Filename: "image.unknown", MediaMetadata: &v1pb.MediaMetadata{Width: proto.Int32(20), Height: proto.Int32(10)}}
spec := &v1pb.UploadAttachmentRequest_Spec{Spec: &v1pb.UploadAttachmentSpec{Attachment: metadata, TotalSize: int64(len(tc.content))}}
initial, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: spec})
require.NoError(t, err)
result, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(initial.UploadId), Data: tc.content, FinishWrite: true})
require.NoError(t, err)
require.Equal(t, "image/jpeg", result.Attachment.Type)
require.True(t, proto.Equal(metadata.MediaMetadata, result.Attachment.MediaMetadata))
uid, err := ExtractAttachmentUIDFromName(result.Attachment.Name)
require.NoError(t, err)
row, err := svc.Store.GetAttachment(ctx, &store.FindAttachment{UID: &uid})
require.NoError(t, err)
content, err := os.ReadFile(row.Reference)
require.NoError(t, err)
if tc.motion {
require.Equal(t, tc.content, content)
require.True(t, result.Attachment.MotionMedia.HasEmbeddedVideo)
} else {
stripped, err := stripImageExif(bytes.NewReader(tc.content), "image/jpeg")
require.NoError(t, err)
require.Equal(t, stripped, content)
}
})
}
}

View file

@ -0,0 +1,111 @@
package v1
import (
"bytes"
"compress/gzip"
"context"
"io"
"net/http"
"net/http/httptest"
"testing"
"connectrpc.com/connect"
"github.com/labstack/echo/v5"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/proto"
v1pb "github.com/usememos/memos/proto/gen/api/v1"
"github.com/usememos/memos/proto/gen/api/v1/apiv1connect"
"github.com/usememos/memos/server/auth"
)
type uploadCountingReader struct {
reader io.Reader
read int
}
func (r *uploadCountingReader) Read(p []byte) (int, error) {
n, err := r.reader.Read(p)
r.read += n
return n, err
}
func TestUploadAttachmentTransports(t *testing.T) {
svc, ctx := newUploadTestService(t)
user, err := svc.fetchCurrentUser(ctx)
require.NoError(t, err)
token, _, err := auth.GenerateAccessTokenV2(user.ID, user.Username, string(user.Role), string(user.RowStatus), []byte(svc.Secret))
require.NoError(t, err)
e := echo.New()
require.NoError(t, svc.RegisterGateway(context.Background(), e))
for _, path := range []string{"/api/v1/attachments:upload", attachmentUploadProcedure} {
t.Run(path, func(t *testing.T) {
call := func(message proto.Message, authenticated bool) *httptest.ResponseRecorder {
t.Helper()
data, err := protojson.Marshal(message)
require.NoError(t, err)
req := httptest.NewRequest(http.MethodPost, path, bytes.NewReader(data))
req.Header.Set("Content-Type", "application/json")
if authenticated {
req.Header.Set("Authorization", "Bearer "+token)
}
rec := httptest.NewRecorder()
e.ServeHTTP(rec, req)
return rec
}
initial := &v1pb.UploadAttachmentRequest{Upload: uploadSpec("file.txt", 3)}
require.Equal(t, http.StatusUnauthorized, call(initial, false).Code)
rec := call(initial, true)
require.Equal(t, http.StatusOK, rec.Code, rec.Body.String())
response := &v1pb.UploadAttachmentResponse{}
require.NoError(t, protojson.Unmarshal(rec.Body.Bytes(), response))
rec = call(&v1pb.UploadAttachmentRequest{Upload: uploadID(response.UploadId), Data: []byte("abc"), FinishWrite: true}, true)
require.Equal(t, http.StatusOK, rec.Code, rec.Body.String())
require.NoError(t, protojson.Unmarshal(rec.Body.Bytes(), response))
require.NotNil(t, response.Attachment)
oversized, err := protojson.Marshal(&v1pb.UploadAttachmentRequest{Data: make([]byte, attachmentUploadRequestLimit)})
require.NoError(t, err)
reader := &uploadCountingReader{reader: bytes.NewReader(oversized)}
req := httptest.NewRequest(http.MethodPost, path, reader)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+token)
rec = httptest.NewRecorder()
e.ServeHTTP(rec, req)
require.GreaterOrEqual(t, rec.Code, 400)
require.LessOrEqual(t, reader.read, attachmentUploadRequestLimit+1, "body must be bounded before decoding")
})
}
t.Run("Connect protobuf and compressed message limit", func(t *testing.T) {
server := httptest.NewTestServer(t, e)
client := apiv1connect.NewAttachmentServiceClient(server.Client(), server.URL)
initial := connect.NewRequest(&v1pb.UploadAttachmentRequest{Upload: uploadSpec("binary.bin", attachmentUploadChunkSize)})
initial.Header().Set("Authorization", "Bearer "+token)
response, err := client.UploadAttachment(context.Background(), initial)
require.NoError(t, err)
last := connect.NewRequest(&v1pb.UploadAttachmentRequest{Upload: uploadID(response.Msg.UploadId), Data: make([]byte, attachmentUploadChunkSize), FinishWrite: true})
last.Header().Set("Authorization", "Bearer "+token)
finished, err := client.UploadAttachment(context.Background(), last)
require.NoError(t, err)
require.EqualValues(t, attachmentUploadChunkSize, finished.Msg.Attachment.Size)
data, err := proto.Marshal(&v1pb.UploadAttachmentRequest{Data: make([]byte, attachmentUploadRequestLimit+1)})
require.NoError(t, err)
var compressed bytes.Buffer
writer := gzip.NewWriter(&compressed)
_, err = writer.Write(data)
require.NoError(t, err)
require.NoError(t, writer.Close())
req := httptest.NewRequest(http.MethodPost, attachmentUploadProcedure, &compressed)
req.Header.Set("Content-Type", "application/proto")
req.Header.Set("Content-Encoding", "gzip")
req.Header.Set("Authorization", "Bearer "+token)
rec := httptest.NewRecorder()
e.ServeHTTP(rec, req)
require.Equal(t, http.StatusTooManyRequests, rec.Code, rec.Body.String())
require.Contains(t, rec.Body.String(), "resource_exhausted")
})
}

View file

@ -595,6 +595,14 @@ func (s *ConnectServiceHandler) DeleteSpaceMember(ctx context.Context, req *conn
// AttachmentService
func (s *ConnectServiceHandler) UploadAttachment(ctx context.Context, req *connect.Request[v1pb.UploadAttachmentRequest]) (*connect.Response[v1pb.UploadAttachmentResponse], error) {
resp, err := s.APIV1Service.UploadAttachment(ctx, req.Msg)
if err != nil {
return nil, convertGRPCError(err)
}
return connect.NewResponse(resp), nil
}
func (s *ConnectServiceHandler) CreateAttachment(ctx context.Context, req *connect.Request[v1pb.CreateAttachmentRequest]) (*connect.Response[v1pb.Attachment], error) {
resp, err := s.APIV1Service.CreateAttachment(ctx, req.Msg)
if err != nil {

View file

@ -21,6 +21,40 @@ func TestS3AttachmentLifecycleAcrossStorageChange(t *testing.T) {
runS3AttachmentLifecycleAcrossStorageChange(t, fake)
}
func TestUploadAttachmentS3(t *testing.T) {
ts := NewTestService(t)
defer ts.Cleanup()
defer ts.Service.CloseAttachmentUploads()
fake := fakes3.New(t, "uploads")
ctx := context.Background()
user, err := ts.CreateRegularUser(ctx, "chunk-uploader")
require.NoError(t, err)
ctx = ts.CreateUserContext(ctx, user.ID)
storage := fakeStorage("upload-s3", "Uploads", fake.Config("uploads"))
upsertS3StorageSetting(ctx, t, ts, storage.Id, storage)
chunk := bytes.Repeat([]byte("x"), 2<<20)
initial, err := ts.Service.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: &v1pb.UploadAttachmentRequest_Spec{Spec: &v1pb.UploadAttachmentSpec{
Attachment: &v1pb.Attachment{Filename: "large.bin", Type: "application/octet-stream"}, TotalSize: int64(3 * len(chunk)),
}}})
require.NoError(t, err)
var result *v1pb.UploadAttachmentResponse
for i := range 3 {
result, err = ts.Service.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{
Upload: &v1pb.UploadAttachmentRequest_UploadId{UploadId: initial.UploadId}, WriteOffset: int64(i * len(chunk)), Data: chunk, FinishWrite: i == 2,
})
require.NoError(t, err)
}
uid, err := apiv1.ExtractAttachmentUIDFromName(result.Attachment.Name)
require.NoError(t, err)
row, err := ts.Store.GetAttachment(ctx, &store.FindAttachment{UID: &uid})
require.NoError(t, err)
require.Empty(t, row.Blob)
require.Equal(t, storepb.AttachmentStorageType_S3, row.StorageType)
content, err := fake.GetObject("uploads", row.Payload.GetS3Object().GetKey())
require.NoError(t, err)
require.Equal(t, bytes.Repeat(chunk, 3), content)
}
func TestS3AttachmentLifecycleAcrossStorageChangeMinIO(t *testing.T) {
server := testminio.New(t, "attachments-old", "attachments-new")
runS3AttachmentLifecycleAcrossStorageChange(t, server)

View file

@ -49,14 +49,14 @@ func TestUpdateMemoTimestamps(t *testing.T) {
}})
require.NoError(t, err)
tc.patch.Name = memo.Name
before := time.Now().Unix()
beforeSec := time.Now().Unix()
updated, err := ts.Service.UpdateMemo(userCtx, &apiv1.UpdateMemoRequest{
Memo: tc.patch, UpdateMask: &fieldmaskpb.FieldMask{Paths: tc.paths},
})
require.NoError(t, err)
require.Equal(t, tc.wantCreate, updated.CreateTime.Seconds)
if tc.autoUpdate {
require.GreaterOrEqual(t, updated.UpdateTime.Seconds, before)
require.GreaterOrEqual(t, updated.UpdateTime.Seconds, beforeSec)
require.LessOrEqual(t, updated.UpdateTime.Seconds, time.Now().Unix())
} else {
require.Equal(t, tc.wantUpdate, updated.UpdateTime.Seconds)

View file

@ -25,6 +25,15 @@ import (
// it derives its own limit from this constant to keep the two gates in lockstep.
const MaxAPIRequestBytes = 256 << 20
// requestBodyLimit returns the request body cap for a procedure. Chunked
// uploads carry at most one chunk per call, so they get a much lower cap.
func requestBodyLimit(procedure string) int64 {
if procedure == attachmentUploadProcedure {
return attachmentUploadRequestLimit
}
return MaxAPIRequestBytes
}
type APIV1Service struct {
v1pb.UnimplementedInstanceServiceServer
v1pb.UnimplementedAuthServiceServer
@ -51,6 +60,7 @@ type APIV1Service struct {
instanceStatsCache instanceStatsCache
linkMetadataFetcher linkMetadataFetcher
attachmentUploads attachmentUploads
}
// NewAPIV1Service creates an API v1 service with its shared dependencies.
@ -132,6 +142,7 @@ func (s *APIV1Service) RegisterGateway(ctx context.Context, echoServer *echo.Ech
if result != nil {
r = r.WithContext(auth.ApplyToContext(ctx, result))
}
r.Body = http.MaxBytesReader(w, r.Body, requestBodyLimit(procedure))
next(w, r, pathParams)
}
@ -172,7 +183,7 @@ func (s *APIV1Service) RegisterGateway(ctx context.Context, echoServer *echo.Ech
gwGroup := echoServer.Group("")
// Register SSE endpoint with same CORS as rest of /api/v1.
RegisterSSERoutes(gwGroup, s.SSEHub, s.Store, s.Secret)
handler := echo.WrapHandler(http.MaxBytesHandler(gwMux, MaxAPIRequestBytes))
handler := echo.WrapHandler(gwMux)
gwGroup.Any("/api/v1/*", handler)
gwGroup.Any("/file/*", handler)
@ -187,10 +198,17 @@ func (s *APIV1Service) RegisterGateway(ctx context.Context, echoServer *echo.Ech
)
connectMux := http.NewServeMux()
connectHandler := NewConnectServiceHandler(s)
connectHandler.RegisterConnectHandlers(connectMux, connectInterceptors, connect.WithReadMaxBytes(MaxAPIRequestBytes))
connectHandler.RegisterConnectHandlers(connectMux, connectInterceptors,
// Bound the decompressed message as well as the wire bytes below.
connect.WithConditionalHandlerOptions(func(spec connect.Spec) []connect.HandlerOption {
return []connect.HandlerOption{connect.WithReadMaxBytes(int(requestBodyLimit(spec.Procedure)))}
}))
connectGroup := echoServer.Group("")
connectGroup.Any("/memos.api.v1.*", echo.WrapHandler(http.MaxBytesHandler(connectMux, MaxAPIRequestBytes)))
connectGroup.Any("/memos.api.v1.*", echo.WrapHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, requestBodyLimit(r.URL.Path))
connectMux.ServeHTTP(w, r)
})))
return nil
}

View file

@ -30,9 +30,9 @@ type Server struct {
Profile *profile.Profile
Store *store.Store
echoServer *echo.Echo
httpServer *http.Server
sseHub *apiv1.SSEHub
echoServer *echo.Echo
httpServer *http.Server
apiV1Service *apiv1.APIV1Service
}
func NewServer(ctx context.Context, profile *profile.Profile, store *store.Store) (*Server, error) {
@ -66,7 +66,7 @@ func NewServer(ctx context.Context, profile *profile.Profile, store *store.Store
frontend.NewFrontendService(profile, store).Serve(ctx, echoServer)
apiV1Service := apiv1.NewAPIV1Service(s.Secret, profile, store)
s.sseHub = apiV1Service.SSEHub
s.apiV1Service = apiV1Service
// Register HTTP file server routes BEFORE gRPC-Gateway to ensure proper range request handling for Safari.
// This uses native HTTP serving (http.ServeContent) instead of gRPC for video/audio files.
@ -127,6 +127,7 @@ func (s *Server) Shutdown(ctx context.Context) {
s.closeLongLivedConnections()
s.shutdownHTTPServer(ctx)
s.apiV1Service.CloseAttachmentUploads()
// Close database connection.
if err := s.Store.Close(); err != nil {
@ -138,9 +139,7 @@ func (s *Server) Shutdown(ctx context.Context) {
func (s *Server) closeLongLivedConnections() {
// Long-lived SSE requests do not finish on their own during http.Server.Shutdown.
if s.sseHub != nil {
s.sseHub.Close()
}
s.apiV1Service.SSEHub.Close()
}
func (s *Server) shutdownHTTPServer(ctx context.Context) {

View file

@ -1,35 +1,74 @@
import { create } from "@bufbuild/protobuf";
import { Code, ConnectError } from "@connectrpc/connect";
import { attachmentServiceClient } from "@/connect";
import type { Attachment } from "@/types/proto/api/v1/attachment_service_pb";
import { AttachmentSchema, MotionMediaSchema } from "@/types/proto/api/v1/attachment_service_pb";
import type { Attachment, UploadAttachmentRequest } from "@/types/proto/api/v1/attachment_service_pb";
import { AttachmentSchema, MotionMediaSchema, UploadAttachmentSpecSchema } from "@/types/proto/api/v1/attachment_service_pb";
import type { LocalFile } from "../types/attachment";
const DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024;
// The server allows eight unfinished uploads per user; stay well under it.
const MAX_CONCURRENT_UPLOADS = 4;
export const uploadService = {
async uploadFile(localFile: LocalFile): Promise<Attachment> {
async uploadFile(localFile: LocalFile, signal?: AbortSignal): Promise<Attachment> {
const { file, motionMedia } = localFile;
const [mediaMetadata, arrayBuffer] = await Promise.all([localFile.mediaMetadata, file.arrayBuffer()]);
const buffer = new Uint8Array(arrayBuffer);
return attachmentServiceClient.createAttachment({
const mediaMetadata = await localFile.mediaMetadata;
const spec = create(UploadAttachmentSpecSchema, {
attachment: create(AttachmentSchema, {
filename: file.name,
size: BigInt(file.size),
type: file.type,
content: buffer,
motionMedia: motionMedia ? create(MotionMediaSchema, motionMedia) : undefined,
mediaMetadata,
}),
totalSize: BigInt(file.size),
});
// The spec call carries no data so that retrying it after a lost response
// can at worst orphan an upload, never create a duplicate attachment.
const initial = await attachmentServiceClient.uploadAttachment({ upload: { case: "spec", value: spec } }, { signal });
if (!initial.uploadId || initial.committedSize !== 0n || initial.maxChunkSize <= 0) {
throw new Error("Invalid upload initialization response");
}
const chunkSize = Math.min(initial.maxChunkSize, DEFAULT_CHUNK_SIZE);
for (let offset = 0; ; ) {
signal?.throwIfAborted();
const end = Math.min(offset + chunkSize, file.size);
const request: Pick<UploadAttachmentRequest, "upload" | "writeOffset" | "data" | "finishWrite"> = {
upload: { case: "uploadId", value: initial.uploadId },
writeOffset: BigInt(offset),
data: new Uint8Array(await file.slice(offset, end).arrayBuffer()),
finishWrite: end === file.size,
};
// Retrying the identical last write is safe even if its response was lost
// after the server finalized the attachment.
let response;
for (let attempt = 0; ; attempt++) {
try {
response = await attachmentServiceClient.uploadAttachment(request, { signal });
break;
} catch (error) {
if (signal?.aborted || ConnectError.from(error).code !== Code.Unavailable || attempt >= 2) throw error;
}
}
if (response.committedSize !== BigInt(end)) {
throw new Error("Unexpected upload offset");
}
if (request.finishWrite) {
if (!response.attachment) throw new Error("Upload completed without an attachment");
return response.attachment;
}
offset = end;
}
},
async uploadFiles(localFiles: LocalFile[]): Promise<Attachment[]> {
if (localFiles.length === 0) return [];
const attachments: Attachment[] = [];
for (const localFile of localFiles) {
attachments.push(await uploadService.uploadFile(localFile));
}
const attachments: Attachment[] = new Array(localFiles.length);
let next = 0;
const worker = async () => {
for (let index = next++; index < localFiles.length; index = next++) {
attachments[index] = await this.uploadFile(localFiles[index]);
}
};
await Promise.all(Array.from({ length: Math.min(MAX_CONCURRENT_UPLOADS, localFiles.length) }, worker));
return attachments;
},
};

File diff suppressed because one or more lines are too long

View file

@ -4,14 +4,14 @@ import type { LocalFile } from "@/components/MemoEditor/types/attachment";
import { MediaMetadataSchema } from "@/types/proto/api/v1/attachment_service_pb";
const mocks = vi.hoisted(() => ({
createAttachment: vi.fn(),
uploadAttachment: vi.fn(),
extractMetadata: vi.fn(),
}));
// The ingest helper lives beside useFileUpload, whose module graph reaches
// AuthContext and the query hooks; stub every client they name-import.
vi.mock("@/connect", () => ({
attachmentServiceClient: { createAttachment: mocks.createAttachment },
attachmentServiceClient: { uploadAttachment: mocks.uploadAttachment },
authServiceClient: {},
userServiceClient: {},
memoViewServiceClient: {},
@ -58,14 +58,19 @@ describe("media metadata at file ingest", () => {
describe("uploadService media metadata", () => {
beforeEach(() => {
mocks.createAttachment.mockImplementation(async ({ attachment }) => attachment);
mocks.uploadAttachment.mockImplementation(async ({ data }) => ({
uploadId: "test-id",
maxChunkSize: 2 * 1024 * 1024,
committedSize: BigInt(data?.length ?? 0),
attachment: data ? { name: "attachments/test" } : undefined,
}));
});
it("submits no metadata for files ingested without any", async () => {
await uploadService.uploadFiles([localImage()]);
expect(mocks.createAttachment).toHaveBeenCalledOnce();
expect(mocks.createAttachment.mock.calls[0][0].attachment.mediaMetadata).toBeUndefined();
expect(mocks.uploadAttachment).toHaveBeenCalledTimes(2);
expect(mocks.uploadAttachment.mock.calls[0][0].upload.value.attachment.mediaMetadata).toBeUndefined();
});
it("submits the metadata extracted at ingest", async () => {
@ -73,14 +78,14 @@ describe("uploadService media metadata", () => {
await uploadService.uploadFiles([localImage(Promise.resolve(metadata))]);
expect(mocks.createAttachment.mock.calls[0][0].attachment.mediaMetadata).toEqual(metadata);
expect(mocks.uploadAttachment.mock.calls[0][0].upload.value.attachment.mediaMetadata).toEqual(metadata);
});
it("continues without metadata when extraction produced no usable values", async () => {
await uploadService.uploadFiles([localImage(Promise.resolve(undefined))]);
expect(mocks.createAttachment).toHaveBeenCalledOnce();
expect(mocks.createAttachment.mock.calls[0][0].attachment.mediaMetadata).toBeUndefined();
expect(mocks.uploadAttachment).toHaveBeenCalledTimes(2);
expect(mocks.uploadAttachment.mock.calls[0][0].upload.value.attachment.mediaMetadata).toBeUndefined();
});
it("submits ingest-time metadata end to end", async () => {
@ -90,6 +95,6 @@ describe("uploadService media metadata", () => {
await uploadService.uploadFiles(toLocalFiles([file], { createBlobUrl, saveMediaMetadata: true }));
expect(mocks.createAttachment.mock.calls[0][0].attachment.mediaMetadata).toEqual(metadata);
expect(mocks.uploadAttachment.mock.calls[0][0].upload.value.attachment.mediaMetadata).toEqual(metadata);
});
});

View file

@ -0,0 +1,98 @@
import { create } from "@bufbuild/protobuf";
import { Code, ConnectError } from "@connectrpc/connect";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { AttachmentSchema } from "@/types/proto/api/v1/attachment_service_pb";
const { uploadAttachment } = vi.hoisted(() => ({ uploadAttachment: vi.fn() }));
vi.mock("@/connect", () => ({ attachmentServiceClient: { uploadAttachment } }));
import { uploadService } from "@/components/MemoEditor/services/uploadService";
const attachment = create(AttachmentSchema, { name: "attachments/uploaded" });
const localFile = (content: string) => ({
file: new File([content], "test.txt", { type: "text/plain" }),
origin: "upload" as const,
previewUrl: "blob:test",
});
describe("chunked attachment uploads", () => {
beforeEach(() => {
uploadAttachment.mockImplementation(async (request) => ({
uploadId: "opaque-id",
maxChunkSize: 3,
committedSize: request.data ? request.writeOffset + BigInt(request.data.length) : 0n,
attachment: request.finishWrite ? attachment : undefined,
}));
});
it("reads only file slices and writes consecutive offsets", async () => {
const local = localFile("abcdefgh");
const wholeFileRead = vi.spyOn(local.file, "arrayBuffer").mockRejectedValue(new Error("must not read the whole file"));
const slice = vi.spyOn(local.file, "slice");
await expect(uploadService.uploadFile(local)).resolves.toEqual(attachment);
expect(wholeFileRead).not.toHaveBeenCalled();
expect(slice.mock.calls).toEqual([
[0, 3],
[3, 6],
[6, 8],
]);
const requests = uploadAttachment.mock.calls.map(([request]) => request);
expect(requests[0].upload.case).toBe("spec");
expect(requests[0].upload.value.totalSize).toBe(8n);
expect(requests[0].upload.value.attachment.content).toHaveLength(0);
expect(requests[0].data).toBeUndefined();
expect(requests.slice(1).map((request) => request.upload)).toEqual(Array(3).fill({ case: "uploadId", value: "opaque-id" }));
expect(requests.slice(1).map((request) => request.writeOffset)).toEqual([0n, 3n, 6n]);
expect(requests.slice(1).map((request) => new TextDecoder().decode(request.data))).toEqual(["abc", "def", "gh"]);
expect(requests.slice(1).map((request) => request.finishWrite)).toEqual([false, false, true]);
});
it("finalizes an empty file", async () => {
await expect(uploadService.uploadFile(localFile(""))).resolves.toEqual(attachment);
expect(uploadAttachment).toHaveBeenCalledTimes(2);
expect(uploadAttachment.mock.calls[1][0]).toMatchObject({ writeOffset: 0n, finishWrite: true });
expect(uploadAttachment.mock.calls[1][0].data).toHaveLength(0);
});
it("retries an identical final chunk after a lost response", async () => {
const original = uploadAttachment.getMockImplementation();
let lost = false;
uploadAttachment.mockImplementation(async (request) => {
if (request.finishWrite && !lost) {
lost = true;
throw new ConnectError("response lost", Code.Unavailable);
}
return original?.(request);
});
await expect(uploadService.uploadFile(localFile("abc"))).resolves.toEqual(attachment);
expect(uploadAttachment.mock.calls[1][0]).toBe(uploadAttachment.mock.calls[2][0]);
});
it("does not retry permission errors", async () => {
uploadAttachment.mockRejectedValueOnce(new ConnectError("denied", Code.PermissionDenied));
await expect(uploadService.uploadFile(localFile("abc"))).rejects.toMatchObject({ code: Code.PermissionDenied });
expect(uploadAttachment).toHaveBeenCalledTimes(1);
});
it("bounds retries and stops on cancellation", async () => {
const initial = { uploadId: "id", committedSize: 0n, maxChunkSize: 3 };
uploadAttachment.mockResolvedValueOnce(initial).mockRejectedValue(new ConnectError("offline", Code.Unavailable));
await expect(uploadService.uploadFile(localFile("abc"))).rejects.toMatchObject({ code: Code.Unavailable });
expect(uploadAttachment).toHaveBeenCalledTimes(4);
uploadAttachment.mockReset().mockResolvedValue(initial);
const controller = new AbortController();
controller.abort();
await expect(uploadService.uploadFile(localFile("abc"), controller.signal)).rejects.toThrow();
expect(uploadAttachment).toHaveBeenCalledTimes(1);
});
it("rejects invalid progress instead of looping or skipping bytes", async () => {
uploadAttachment.mockResolvedValue({ uploadId: "id", committedSize: 0n, maxChunkSize: 3 });
await expect(uploadService.uploadFile(localFile("abcdef"))).rejects.toThrow("Unexpected upload offset");
expect(uploadAttachment).toHaveBeenCalledTimes(2);
});
});