diff --git a/internal/motionphoto/motionphoto.go b/internal/motionphoto/motionphoto.go index 2bc0b1ab..267e240b 100644 --- a/internal/motionphoto/motionphoto.go +++ b/internal/motionphoto/motionphoto.go @@ -20,25 +20,11 @@ var ( const maxMetadataScanBytes = 256 * 1024 +// DetectJPEG detects embedded video in an in-memory JPEG. func DetectJPEG(blob []byte) *Detection { - if len(blob) < 16 || !bytes.HasPrefix(blob, []byte{0xFF, 0xD8}) { - return nil - } - - text := string(blob[:min(len(blob), maxMetadataScanBytes)]) - if !motionPhotoMarkerRegex.MatchString(text) { - return nil - } - - videoStart := detectVideoStart(blob, text) - if videoStart < 0 || videoStart >= len(blob) { - return nil - } - - return &Detection{ - VideoStart: videoStart, - PresentationTimestampUs: parsePresentationTimestampUs(text), - } + // bytes.Reader never fails within bounds, so the error is unreachable. + detection, _ := DetectJPEGReader(bytes.NewReader(blob), int64(len(blob))) + return detection } func ExtractVideo(blob []byte) ([]byte, *Detection) { @@ -55,19 +41,6 @@ func ExtractVideo(blob []byte) ([]byte, *Detection) { return videoBlob, detection } -func detectVideoStart(blob []byte, text string) int { - if matches := microVideoOffsetRegex.FindStringSubmatch(text); len(matches) == 2 { - if offset, err := strconv.Atoi(matches[1]); err == nil && offset > 0 && offset < len(blob) { - start := len(blob) - offset - if looksLikeMP4(blob[start:]) { - return start - } - } - } - - return findEmbeddedMP4Start(blob) -} - func parsePresentationTimestampUs(text string) int64 { matches := presentationRegex.FindStringSubmatch(text) if len(matches) != 2 { diff --git a/internal/motionphoto/reader.go b/internal/motionphoto/reader.go new file mode 100644 index 00000000..d1e57314 --- /dev/null +++ b/internal/motionphoto/reader.go @@ -0,0 +1,56 @@ +package motionphoto + +import ( + "bytes" + "io" + "strconv" +) + +// DetectJPEGReader detects embedded video without loading the JPEG into memory. +// It reads at most the metadata header plus the blocks needed to locate the video. +func DetectJPEGReader(reader io.ReaderAt, size int64) (*Detection, error) { + if size < 16 { + return nil, nil + } + header := make([]byte, min(size, maxMetadataScanBytes)) + if _, err := reader.ReadAt(header, 0); err != nil { + return nil, err + } + if !bytes.HasPrefix(header, []byte{0xFF, 0xD8}) || !motionPhotoMarkerRegex.Match(header) { + return nil, nil + } + text := string(header) + makeDetection := func(start int64) *Detection { + return &Detection{VideoStart: int(start), PresentationTimestampUs: parsePresentationTimestampUs(text)} + } + if matches := microVideoOffsetRegex.FindStringSubmatch(text); len(matches) == 2 { + if offset, err := strconv.ParseInt(matches[1], 10, 64); err == nil && offset >= 12 && offset < size { + var probe [12]byte + if _, err := reader.ReadAt(probe[:], size-offset); err != nil { + return nil, err + } + if looksLikeMP4(probe[:]) { + return makeDetection(size - offset), nil + } + } + } + + // Fall back to the last valid ftyp box, scanning backwards with overlap so + // a box header crossing a block boundary is still found. + buffer := make([]byte, 64*1024) + for end := size; end >= 12; { + start := max(int64(0), end-int64(len(buffer))) + block := buffer[:end-start] + if _, err := reader.ReadAt(block, start); err != nil { + return nil, err + } + if index := findEmbeddedMP4Start(block); index >= 0 { + return makeDetection(start + int64(index)), nil + } + if start == 0 { + break + } + end = start + 11 + } + return nil, nil +} diff --git a/internal/motionphoto/reader_test.go b/internal/motionphoto/reader_test.go new file mode 100644 index 00000000..6ff466d5 --- /dev/null +++ b/internal/motionphoto/reader_test.go @@ -0,0 +1,45 @@ +package motionphoto + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/usememos/memos/internal/testutil" +) + +func TestDetectJPEGReader(t *testing.T) { + for _, blob := range [][]byte{nil, []byte("not a JPEG"), []byte("\xff\xd8 no motion marker here at all")} { + detected, err := DetectJPEGReader(bytes.NewReader(blob), int64(len(blob))) + require.NoError(t, err) + require.Nil(t, detected) + } + fixture := testutil.BuildMotionPhotoJPEG() + detected, err := DetectJPEGReader(bytes.NewReader(fixture), int64(len(fixture))) + require.NoError(t, err) + require.NotNil(t, detected) + require.True(t, looksLikeMP4(fixture[detected.VideoStart:])) + require.EqualValues(t, 123456, detected.PresentationTimestampUs) + + // Exercise MP4 headers straddling both sides of the backwards scan boundary. + for offset := -12; offset <= 12; offset++ { + blob := make([]byte, 200_000) + copy(blob, []byte("\xff\xd8 Camera:MotionPhoto=\"1\"")) + start := len(blob) - 64*1024 + offset + copy(blob[start:], []byte{0, 0, 0, 16, 'f', 't', 'y', 'p', 'm', 'p', '4', '2'}) + detected, err := DetectJPEGReader(bytes.NewReader(blob), int64(len(blob))) + require.NoError(t, err) + require.NotNil(t, detected, "offset %d", offset) + require.Equal(t, start, detected.VideoStart, "offset %d", offset) + } + + // Prefer the explicit MicroVideoOffset even when a later ftyp box exists. + blob := make([]byte, 1024) + copy(blob, []byte("\xff\xd8 Camera:MotionPhoto=\"1\" Camera:MicroVideoOffset=\"128\"")) + copy(blob[896:], []byte{0, 0, 0, 16, 'f', 't', 'y', 'p', 'm', 'p', '4', '2'}) + copy(blob[1000:], []byte{0, 0, 0, 16, 'f', 't', 'y', 'p', 'm', 'p', '4', '2'}) + detected, err = DetectJPEGReader(bytes.NewReader(blob), int64(len(blob))) + require.NoError(t, err) + require.Equal(t, 896, detected.VideoStart) +} diff --git a/proto/api/v1/attachment_service.proto b/proto/api/v1/attachment_service.proto index f489ff60..884b6cac 100644 --- a/proto/api/v1/attachment_service.proto +++ b/proto/api/v1/attachment_service.proto @@ -21,6 +21,16 @@ service AttachmentService { }; option (google.api.method_signature) = "attachment"; } + // UploadAttachment uploads a file in bounded chunks. The first call carries + // the spec and returns an upload_id; later calls carry that upload_id. + // Uploads are bound to the authenticated user, expire after 30 minutes of + // inactivity, and do not survive a server restart. + rpc UploadAttachment(UploadAttachmentRequest) returns (UploadAttachmentResponse) { + option (google.api.http) = { + post: "/api/v1/attachments:upload" + body: "*" + }; + } // ListAttachments lists all attachments. rpc ListAttachments(ListAttachmentsRequest) returns (ListAttachmentsResponse) { option (google.api.http) = {get: "/api/v1/attachments"}; @@ -179,6 +189,61 @@ message CreateAttachmentRequest { string attachment_id = 2 [(google.api.field_behavior) = OPTIONAL]; } +message UploadAttachmentRequest { + // Required. Start a new upload or continue an existing one. + oneof upload { + // Starts a new upload. The same call may also carry data and finish_write. + UploadAttachmentSpec spec = 1; + + // Continues the upload identified by a previous response. + string upload_id = 2; + } + + // Required. Zero-based byte offset at which data is written. Must equal the + // committed size, except when retrying the most recently accepted chunk + // with identical bytes and offset, which is accepted without writing again. + int64 write_offset = 3 [(google.api.field_behavior) = REQUIRED]; + + // Optional. File bytes, at most max_chunk_size long. With no data and + // finish_write false, the call reports progress without writing, and + // write_offset is ignored. + bytes data = 4 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Finalize the upload after writing data. The committed size must + // then equal total_size. Any later call for the same upload_id returns the + // created attachment. + bool finish_write = 5 [(google.api.field_behavior) = OPTIONAL]; +} + +message UploadAttachmentSpec { + // Required. Metadata for the attachment to create. content must be empty; + // file bytes are sent in data. + Attachment attachment = 1 [(google.api.field_behavior) = REQUIRED]; + + // Optional. The attachment ID to use for this attachment. + // If empty, a unique ID will be generated. + // Format: ^[a-zA-Z0-9]([a-zA-Z0-9-]{0,34}[a-zA-Z0-9])?$ + string attachment_id = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Total size of the file in bytes before media processing. + // Zero represents an empty file. + int64 total_size = 3 [(google.api.field_behavior) = OPTIONAL]; +} + +message UploadAttachmentResponse { + // Opaque ID for subsequent calls. This is not a resource name. + string upload_id = 1; + + // Number of file bytes committed so far. + int64 committed_size = 2; + + // Set once the upload has been finalized. + Attachment attachment = 3; + + // Maximum number of data bytes accepted in one call. + int32 max_chunk_size = 4; +} + message ListAttachmentsRequest { // Optional. The maximum number of attachments to return. // The service may return fewer than this value. diff --git a/proto/gen/api/v1/apiv1connect/attachment_service.connect.go b/proto/gen/api/v1/apiv1connect/attachment_service.connect.go index a5dda3af..5424a142 100644 --- a/proto/gen/api/v1/apiv1connect/attachment_service.connect.go +++ b/proto/gen/api/v1/apiv1connect/attachment_service.connect.go @@ -37,6 +37,9 @@ const ( // AttachmentServiceCreateAttachmentProcedure is the fully-qualified name of the AttachmentService's // CreateAttachment RPC. AttachmentServiceCreateAttachmentProcedure = "/memos.api.v1.AttachmentService/CreateAttachment" + // AttachmentServiceUploadAttachmentProcedure is the fully-qualified name of the AttachmentService's + // UploadAttachment RPC. + AttachmentServiceUploadAttachmentProcedure = "/memos.api.v1.AttachmentService/UploadAttachment" // AttachmentServiceListAttachmentsProcedure is the fully-qualified name of the AttachmentService's // ListAttachments RPC. AttachmentServiceListAttachmentsProcedure = "/memos.api.v1.AttachmentService/ListAttachments" @@ -58,6 +61,11 @@ const ( type AttachmentServiceClient interface { // CreateAttachment creates a new attachment. CreateAttachment(context.Context, *connect.Request[v1.CreateAttachmentRequest]) (*connect.Response[v1.Attachment], error) + // UploadAttachment uploads a file in bounded chunks. The first call carries + // the spec and returns an upload_id; later calls carry that upload_id. + // Uploads are bound to the authenticated user, expire after 30 minutes of + // inactivity, and do not survive a server restart. + UploadAttachment(context.Context, *connect.Request[v1.UploadAttachmentRequest]) (*connect.Response[v1.UploadAttachmentResponse], error) // ListAttachments lists all attachments. ListAttachments(context.Context, *connect.Request[v1.ListAttachmentsRequest]) (*connect.Response[v1.ListAttachmentsResponse], error) // GetAttachment returns an attachment by name. @@ -87,6 +95,12 @@ func NewAttachmentServiceClient(httpClient connect.HTTPClient, baseURL string, o connect.WithSchema(attachmentServiceMethods.ByName("CreateAttachment")), connect.WithClientOptions(opts...), ), + uploadAttachment: connect.NewClient[v1.UploadAttachmentRequest, v1.UploadAttachmentResponse]( + httpClient, + baseURL+AttachmentServiceUploadAttachmentProcedure, + connect.WithSchema(attachmentServiceMethods.ByName("UploadAttachment")), + connect.WithClientOptions(opts...), + ), listAttachments: connect.NewClient[v1.ListAttachmentsRequest, v1.ListAttachmentsResponse]( httpClient, baseURL+AttachmentServiceListAttachmentsProcedure, @@ -123,6 +137,7 @@ func NewAttachmentServiceClient(httpClient connect.HTTPClient, baseURL string, o // attachmentServiceClient implements AttachmentServiceClient. type attachmentServiceClient struct { createAttachment *connect.Client[v1.CreateAttachmentRequest, v1.Attachment] + uploadAttachment *connect.Client[v1.UploadAttachmentRequest, v1.UploadAttachmentResponse] listAttachments *connect.Client[v1.ListAttachmentsRequest, v1.ListAttachmentsResponse] getAttachment *connect.Client[v1.GetAttachmentRequest, v1.Attachment] updateAttachment *connect.Client[v1.UpdateAttachmentRequest, v1.Attachment] @@ -135,6 +150,11 @@ func (c *attachmentServiceClient) CreateAttachment(ctx context.Context, req *con return c.createAttachment.CallUnary(ctx, req) } +// UploadAttachment calls memos.api.v1.AttachmentService.UploadAttachment. +func (c *attachmentServiceClient) UploadAttachment(ctx context.Context, req *connect.Request[v1.UploadAttachmentRequest]) (*connect.Response[v1.UploadAttachmentResponse], error) { + return c.uploadAttachment.CallUnary(ctx, req) +} + // ListAttachments calls memos.api.v1.AttachmentService.ListAttachments. func (c *attachmentServiceClient) ListAttachments(ctx context.Context, req *connect.Request[v1.ListAttachmentsRequest]) (*connect.Response[v1.ListAttachmentsResponse], error) { return c.listAttachments.CallUnary(ctx, req) @@ -164,6 +184,11 @@ func (c *attachmentServiceClient) BatchDeleteAttachments(ctx context.Context, re type AttachmentServiceHandler interface { // CreateAttachment creates a new attachment. CreateAttachment(context.Context, *connect.Request[v1.CreateAttachmentRequest]) (*connect.Response[v1.Attachment], error) + // UploadAttachment uploads a file in bounded chunks. The first call carries + // the spec and returns an upload_id; later calls carry that upload_id. + // Uploads are bound to the authenticated user, expire after 30 minutes of + // inactivity, and do not survive a server restart. + UploadAttachment(context.Context, *connect.Request[v1.UploadAttachmentRequest]) (*connect.Response[v1.UploadAttachmentResponse], error) // ListAttachments lists all attachments. ListAttachments(context.Context, *connect.Request[v1.ListAttachmentsRequest]) (*connect.Response[v1.ListAttachmentsResponse], error) // GetAttachment returns an attachment by name. @@ -189,6 +214,12 @@ func NewAttachmentServiceHandler(svc AttachmentServiceHandler, opts ...connect.H connect.WithSchema(attachmentServiceMethods.ByName("CreateAttachment")), connect.WithHandlerOptions(opts...), ) + attachmentServiceUploadAttachmentHandler := connect.NewUnaryHandler( + AttachmentServiceUploadAttachmentProcedure, + svc.UploadAttachment, + connect.WithSchema(attachmentServiceMethods.ByName("UploadAttachment")), + connect.WithHandlerOptions(opts...), + ) attachmentServiceListAttachmentsHandler := connect.NewUnaryHandler( AttachmentServiceListAttachmentsProcedure, svc.ListAttachments, @@ -223,6 +254,8 @@ func NewAttachmentServiceHandler(svc AttachmentServiceHandler, opts ...connect.H switch r.URL.Path { case AttachmentServiceCreateAttachmentProcedure: attachmentServiceCreateAttachmentHandler.ServeHTTP(w, r) + case AttachmentServiceUploadAttachmentProcedure: + attachmentServiceUploadAttachmentHandler.ServeHTTP(w, r) case AttachmentServiceListAttachmentsProcedure: attachmentServiceListAttachmentsHandler.ServeHTTP(w, r) case AttachmentServiceGetAttachmentProcedure: @@ -246,6 +279,10 @@ func (UnimplementedAttachmentServiceHandler) CreateAttachment(context.Context, * return nil, connect.NewError(connect.CodeUnimplemented, errors.New("memos.api.v1.AttachmentService.CreateAttachment is not implemented")) } +func (UnimplementedAttachmentServiceHandler) UploadAttachment(context.Context, *connect.Request[v1.UploadAttachmentRequest]) (*connect.Response[v1.UploadAttachmentResponse], error) { + return nil, connect.NewError(connect.CodeUnimplemented, errors.New("memos.api.v1.AttachmentService.UploadAttachment is not implemented")) +} + func (UnimplementedAttachmentServiceHandler) ListAttachments(context.Context, *connect.Request[v1.ListAttachmentsRequest]) (*connect.Response[v1.ListAttachmentsResponse], error) { return nil, connect.NewError(connect.CodeUnimplemented, errors.New("memos.api.v1.AttachmentService.ListAttachments is not implemented")) } diff --git a/proto/gen/api/v1/attachment_service.pb.go b/proto/gen/api/v1/attachment_service.pb.go index b7d9c97c..9eeba093 100644 --- a/proto/gen/api/v1/attachment_service.pb.go +++ b/proto/gen/api/v1/attachment_service.pb.go @@ -770,6 +770,264 @@ func (x *CreateAttachmentRequest) GetAttachmentId() string { return "" } +type UploadAttachmentRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Required. Start a new upload or continue an existing one. + // + // Types that are valid to be assigned to Upload: + // + // *UploadAttachmentRequest_Spec + // *UploadAttachmentRequest_UploadId + Upload isUploadAttachmentRequest_Upload `protobuf_oneof:"upload"` + // Required. Zero-based byte offset at which data is written. Must equal the + // committed size, except when retrying the most recently accepted chunk + // with identical bytes and offset, which is accepted without writing again. + WriteOffset int64 `protobuf:"varint,3,opt,name=write_offset,json=writeOffset,proto3" json:"write_offset,omitempty"` + // Optional. File bytes, at most max_chunk_size long. With no data and + // finish_write false, the call reports progress without writing, and + // write_offset is ignored. + Data []byte `protobuf:"bytes,4,opt,name=data,proto3" json:"data,omitempty"` + // Optional. Finalize the upload after writing data. The committed size must + // then equal total_size. Any later call for the same upload_id returns the + // created attachment. + FinishWrite bool `protobuf:"varint,5,opt,name=finish_write,json=finishWrite,proto3" json:"finish_write,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *UploadAttachmentRequest) Reset() { + *x = UploadAttachmentRequest{} + mi := &file_api_v1_attachment_service_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *UploadAttachmentRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UploadAttachmentRequest) ProtoMessage() {} + +func (x *UploadAttachmentRequest) ProtoReflect() protoreflect.Message { + mi := &file_api_v1_attachment_service_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UploadAttachmentRequest.ProtoReflect.Descriptor instead. +func (*UploadAttachmentRequest) Descriptor() ([]byte, []int) { + return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{8} +} + +func (x *UploadAttachmentRequest) GetUpload() isUploadAttachmentRequest_Upload { + if x != nil { + return x.Upload + } + return nil +} + +func (x *UploadAttachmentRequest) GetSpec() *UploadAttachmentSpec { + if x != nil { + if x, ok := x.Upload.(*UploadAttachmentRequest_Spec); ok { + return x.Spec + } + } + return nil +} + +func (x *UploadAttachmentRequest) GetUploadId() string { + if x != nil { + if x, ok := x.Upload.(*UploadAttachmentRequest_UploadId); ok { + return x.UploadId + } + } + return "" +} + +func (x *UploadAttachmentRequest) GetWriteOffset() int64 { + if x != nil { + return x.WriteOffset + } + return 0 +} + +func (x *UploadAttachmentRequest) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +func (x *UploadAttachmentRequest) GetFinishWrite() bool { + if x != nil { + return x.FinishWrite + } + return false +} + +type isUploadAttachmentRequest_Upload interface { + isUploadAttachmentRequest_Upload() +} + +type UploadAttachmentRequest_Spec struct { + // Starts a new upload. The same call may also carry data and finish_write. + Spec *UploadAttachmentSpec `protobuf:"bytes,1,opt,name=spec,proto3,oneof"` +} + +type UploadAttachmentRequest_UploadId struct { + // Continues the upload identified by a previous response. + UploadId string `protobuf:"bytes,2,opt,name=upload_id,json=uploadId,proto3,oneof"` +} + +func (*UploadAttachmentRequest_Spec) isUploadAttachmentRequest_Upload() {} + +func (*UploadAttachmentRequest_UploadId) isUploadAttachmentRequest_Upload() {} + +type UploadAttachmentSpec struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Required. Metadata for the attachment to create. content must be empty; + // file bytes are sent in data. + Attachment *Attachment `protobuf:"bytes,1,opt,name=attachment,proto3" json:"attachment,omitempty"` + // Optional. The attachment ID to use for this attachment. + // If empty, a unique ID will be generated. + // Format: ^[a-zA-Z0-9]([a-zA-Z0-9-]{0,34}[a-zA-Z0-9])?$ + AttachmentId string `protobuf:"bytes,2,opt,name=attachment_id,json=attachmentId,proto3" json:"attachment_id,omitempty"` + // Optional. Total size of the file in bytes before media processing. + // Zero represents an empty file. + TotalSize int64 `protobuf:"varint,3,opt,name=total_size,json=totalSize,proto3" json:"total_size,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *UploadAttachmentSpec) Reset() { + *x = UploadAttachmentSpec{} + mi := &file_api_v1_attachment_service_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *UploadAttachmentSpec) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UploadAttachmentSpec) ProtoMessage() {} + +func (x *UploadAttachmentSpec) ProtoReflect() protoreflect.Message { + mi := &file_api_v1_attachment_service_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UploadAttachmentSpec.ProtoReflect.Descriptor instead. +func (*UploadAttachmentSpec) Descriptor() ([]byte, []int) { + return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{9} +} + +func (x *UploadAttachmentSpec) GetAttachment() *Attachment { + if x != nil { + return x.Attachment + } + return nil +} + +func (x *UploadAttachmentSpec) GetAttachmentId() string { + if x != nil { + return x.AttachmentId + } + return "" +} + +func (x *UploadAttachmentSpec) GetTotalSize() int64 { + if x != nil { + return x.TotalSize + } + return 0 +} + +type UploadAttachmentResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Opaque ID for subsequent calls. This is not a resource name. + UploadId string `protobuf:"bytes,1,opt,name=upload_id,json=uploadId,proto3" json:"upload_id,omitempty"` + // Number of file bytes committed so far. + CommittedSize int64 `protobuf:"varint,2,opt,name=committed_size,json=committedSize,proto3" json:"committed_size,omitempty"` + // Set once the upload has been finalized. + Attachment *Attachment `protobuf:"bytes,3,opt,name=attachment,proto3" json:"attachment,omitempty"` + // Maximum number of data bytes accepted in one call. + MaxChunkSize int32 `protobuf:"varint,4,opt,name=max_chunk_size,json=maxChunkSize,proto3" json:"max_chunk_size,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *UploadAttachmentResponse) Reset() { + *x = UploadAttachmentResponse{} + mi := &file_api_v1_attachment_service_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *UploadAttachmentResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UploadAttachmentResponse) ProtoMessage() {} + +func (x *UploadAttachmentResponse) ProtoReflect() protoreflect.Message { + mi := &file_api_v1_attachment_service_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UploadAttachmentResponse.ProtoReflect.Descriptor instead. +func (*UploadAttachmentResponse) Descriptor() ([]byte, []int) { + return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{10} +} + +func (x *UploadAttachmentResponse) GetUploadId() string { + if x != nil { + return x.UploadId + } + return "" +} + +func (x *UploadAttachmentResponse) GetCommittedSize() int64 { + if x != nil { + return x.CommittedSize + } + return 0 +} + +func (x *UploadAttachmentResponse) GetAttachment() *Attachment { + if x != nil { + return x.Attachment + } + return nil +} + +func (x *UploadAttachmentResponse) GetMaxChunkSize() int32 { + if x != nil { + return x.MaxChunkSize + } + return 0 +} + type ListAttachmentsRequest struct { state protoimpl.MessageState `protogen:"open.v1"` // Optional. The maximum number of attachments to return. @@ -798,7 +1056,7 @@ type ListAttachmentsRequest struct { func (x *ListAttachmentsRequest) Reset() { *x = ListAttachmentsRequest{} - mi := &file_api_v1_attachment_service_proto_msgTypes[8] + mi := &file_api_v1_attachment_service_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -810,7 +1068,7 @@ func (x *ListAttachmentsRequest) String() string { func (*ListAttachmentsRequest) ProtoMessage() {} func (x *ListAttachmentsRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_attachment_service_proto_msgTypes[8] + mi := &file_api_v1_attachment_service_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -823,7 +1081,7 @@ func (x *ListAttachmentsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListAttachmentsRequest.ProtoReflect.Descriptor instead. func (*ListAttachmentsRequest) Descriptor() ([]byte, []int) { - return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{8} + return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{11} } func (x *ListAttachmentsRequest) GetPageSize() int32 { @@ -867,7 +1125,7 @@ type ListAttachmentsResponse struct { func (x *ListAttachmentsResponse) Reset() { *x = ListAttachmentsResponse{} - mi := &file_api_v1_attachment_service_proto_msgTypes[9] + mi := &file_api_v1_attachment_service_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -879,7 +1137,7 @@ func (x *ListAttachmentsResponse) String() string { func (*ListAttachmentsResponse) ProtoMessage() {} func (x *ListAttachmentsResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_attachment_service_proto_msgTypes[9] + mi := &file_api_v1_attachment_service_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -892,7 +1150,7 @@ func (x *ListAttachmentsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListAttachmentsResponse.ProtoReflect.Descriptor instead. func (*ListAttachmentsResponse) Descriptor() ([]byte, []int) { - return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{9} + return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{12} } func (x *ListAttachmentsResponse) GetAttachments() []*Attachment { @@ -920,7 +1178,7 @@ type GetAttachmentRequest struct { func (x *GetAttachmentRequest) Reset() { *x = GetAttachmentRequest{} - mi := &file_api_v1_attachment_service_proto_msgTypes[10] + mi := &file_api_v1_attachment_service_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -932,7 +1190,7 @@ func (x *GetAttachmentRequest) String() string { func (*GetAttachmentRequest) ProtoMessage() {} func (x *GetAttachmentRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_attachment_service_proto_msgTypes[10] + mi := &file_api_v1_attachment_service_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -945,7 +1203,7 @@ func (x *GetAttachmentRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetAttachmentRequest.ProtoReflect.Descriptor instead. func (*GetAttachmentRequest) Descriptor() ([]byte, []int) { - return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{10} + return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{13} } func (x *GetAttachmentRequest) GetName() string { @@ -967,7 +1225,7 @@ type UpdateAttachmentRequest struct { func (x *UpdateAttachmentRequest) Reset() { *x = UpdateAttachmentRequest{} - mi := &file_api_v1_attachment_service_proto_msgTypes[11] + mi := &file_api_v1_attachment_service_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -979,7 +1237,7 @@ func (x *UpdateAttachmentRequest) String() string { func (*UpdateAttachmentRequest) ProtoMessage() {} func (x *UpdateAttachmentRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_attachment_service_proto_msgTypes[11] + mi := &file_api_v1_attachment_service_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -992,7 +1250,7 @@ func (x *UpdateAttachmentRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateAttachmentRequest.ProtoReflect.Descriptor instead. func (*UpdateAttachmentRequest) Descriptor() ([]byte, []int) { - return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{11} + return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{14} } func (x *UpdateAttachmentRequest) GetAttachment() *Attachment { @@ -1020,7 +1278,7 @@ type DeleteAttachmentRequest struct { func (x *DeleteAttachmentRequest) Reset() { *x = DeleteAttachmentRequest{} - mi := &file_api_v1_attachment_service_proto_msgTypes[12] + mi := &file_api_v1_attachment_service_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1032,7 +1290,7 @@ func (x *DeleteAttachmentRequest) String() string { func (*DeleteAttachmentRequest) ProtoMessage() {} func (x *DeleteAttachmentRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_attachment_service_proto_msgTypes[12] + mi := &file_api_v1_attachment_service_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1045,7 +1303,7 @@ func (x *DeleteAttachmentRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteAttachmentRequest.ProtoReflect.Descriptor instead. func (*DeleteAttachmentRequest) Descriptor() ([]byte, []int) { - return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{12} + return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{15} } func (x *DeleteAttachmentRequest) GetName() string { @@ -1064,7 +1322,7 @@ type BatchDeleteAttachmentsRequest struct { func (x *BatchDeleteAttachmentsRequest) Reset() { *x = BatchDeleteAttachmentsRequest{} - mi := &file_api_v1_attachment_service_proto_msgTypes[13] + mi := &file_api_v1_attachment_service_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1076,7 +1334,7 @@ func (x *BatchDeleteAttachmentsRequest) String() string { func (*BatchDeleteAttachmentsRequest) ProtoMessage() {} func (x *BatchDeleteAttachmentsRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_attachment_service_proto_msgTypes[13] + mi := &file_api_v1_attachment_service_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1089,7 +1347,7 @@ func (x *BatchDeleteAttachmentsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use BatchDeleteAttachmentsRequest.ProtoReflect.Descriptor instead. func (*BatchDeleteAttachmentsRequest) Descriptor() ([]byte, []int) { - return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{13} + return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{16} } func (x *BatchDeleteAttachmentsRequest) GetNames() []string { @@ -1174,7 +1432,28 @@ const file_api_v1_attachment_service_proto_rawDesc = "" + "\n" + "attachment\x18\x01 \x01(\v2\x18.memos.api.v1.AttachmentB\x03\xe0A\x02R\n" + "attachment\x12(\n" + - "\rattachment_id\x18\x02 \x01(\tB\x03\xe0A\x01R\fattachmentId\"\xba\x01\n" + + "\rattachment_id\x18\x02 \x01(\tB\x03\xe0A\x01R\fattachmentId\"\xe5\x01\n" + + "\x17UploadAttachmentRequest\x128\n" + + "\x04spec\x18\x01 \x01(\v2\".memos.api.v1.UploadAttachmentSpecH\x00R\x04spec\x12\x1d\n" + + "\tupload_id\x18\x02 \x01(\tH\x00R\buploadId\x12&\n" + + "\fwrite_offset\x18\x03 \x01(\x03B\x03\xe0A\x02R\vwriteOffset\x12\x17\n" + + "\x04data\x18\x04 \x01(\fB\x03\xe0A\x01R\x04data\x12&\n" + + "\ffinish_write\x18\x05 \x01(\bB\x03\xe0A\x01R\vfinishWriteB\b\n" + + "\x06upload\"\xa3\x01\n" + + "\x14UploadAttachmentSpec\x12=\n" + + "\n" + + "attachment\x18\x01 \x01(\v2\x18.memos.api.v1.AttachmentB\x03\xe0A\x02R\n" + + "attachment\x12(\n" + + "\rattachment_id\x18\x02 \x01(\tB\x03\xe0A\x01R\fattachmentId\x12\"\n" + + "\n" + + "total_size\x18\x03 \x01(\x03B\x03\xe0A\x01R\ttotalSize\"\xbe\x01\n" + + "\x18UploadAttachmentResponse\x12\x1b\n" + + "\tupload_id\x18\x01 \x01(\tR\buploadId\x12%\n" + + "\x0ecommitted_size\x18\x02 \x01(\x03R\rcommittedSize\x128\n" + + "\n" + + "attachment\x18\x03 \x01(\v2\x18.memos.api.v1.AttachmentR\n" + + "attachment\x12$\n" + + "\x0emax_chunk_size\x18\x04 \x01(\x05R\fmaxChunkSize\"\xba\x01\n" + "\x16ListAttachmentsRequest\x12 \n" + "\tpage_size\x18\x01 \x01(\x05B\x03\xe0A\x01R\bpageSize\x12\"\n" + "\n" + @@ -1207,11 +1486,12 @@ const file_api_v1_attachment_service_proto_rawDesc = "" + "\x1dMOTION_MEDIA_ROLE_UNSPECIFIED\x10\x00\x12\t\n" + "\x05STILL\x10\x01\x12\t\n" + "\x05VIDEO\x10\x02\x12\r\n" + - "\tCONTAINER\x10\x032\xd0\x06\n" + + "\tCONTAINER\x10\x032\xdb\a\n" + "\x11AttachmentService\x12\x89\x01\n" + "\x10CreateAttachment\x12%.memos.api.v1.CreateAttachmentRequest\x1a\x18.memos.api.v1.Attachment\"4\xdaA\n" + "attachment\x82\xd3\xe4\x93\x02!:\n" + - "attachment\"\x13/api/v1/attachments\x12{\n" + + "attachment\"\x13/api/v1/attachments\x12\x88\x01\n" + + "\x10UploadAttachment\x12%.memos.api.v1.UploadAttachmentRequest\x1a&.memos.api.v1.UploadAttachmentResponse\"%\x82\xd3\xe4\x93\x02\x1f:\x01*\"\x1a/api/v1/attachments:upload\x12{\n" + "\x0fListAttachments\x12$.memos.api.v1.ListAttachmentsRequest\x1a%.memos.api.v1.ListAttachmentsResponse\"\x1b\x82\xd3\xe4\x93\x02\x15\x12\x13/api/v1/attachments\x12z\n" + "\rGetAttachment\x12\".memos.api.v1.GetAttachmentRequest\x1a\x18.memos.api.v1.Attachment\"+\xdaA\x04name\x82\xd3\xe4\x93\x02\x1e\x12\x1c/api/v1/{name=attachments/*}\x12\xa9\x01\n" + "\x10UpdateAttachment\x12%.memos.api.v1.UpdateAttachmentRequest\x1a\x18.memos.api.v1.Attachment\"T\xdaA\x16attachment,update_mask\x82\xd3\xe4\x93\x025:\n" + @@ -1233,7 +1513,7 @@ func file_api_v1_attachment_service_proto_rawDescGZIP() []byte { } var file_api_v1_attachment_service_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_api_v1_attachment_service_proto_msgTypes = make([]protoimpl.MessageInfo, 14) +var file_api_v1_attachment_service_proto_msgTypes = make([]protoimpl.MessageInfo, 17) var file_api_v1_attachment_service_proto_goTypes = []any{ (MotionMediaFamily)(0), // 0: memos.api.v1.MotionMediaFamily (MotionMediaRole)(0), // 1: memos.api.v1.MotionMediaRole @@ -1245,15 +1525,18 @@ var file_api_v1_attachment_service_proto_goTypes = []any{ (*VideoMetadata)(nil), // 7: memos.api.v1.VideoMetadata (*Attachment)(nil), // 8: memos.api.v1.Attachment (*CreateAttachmentRequest)(nil), // 9: memos.api.v1.CreateAttachmentRequest - (*ListAttachmentsRequest)(nil), // 10: memos.api.v1.ListAttachmentsRequest - (*ListAttachmentsResponse)(nil), // 11: memos.api.v1.ListAttachmentsResponse - (*GetAttachmentRequest)(nil), // 12: memos.api.v1.GetAttachmentRequest - (*UpdateAttachmentRequest)(nil), // 13: memos.api.v1.UpdateAttachmentRequest - (*DeleteAttachmentRequest)(nil), // 14: memos.api.v1.DeleteAttachmentRequest - (*BatchDeleteAttachmentsRequest)(nil), // 15: memos.api.v1.BatchDeleteAttachmentsRequest - (*timestamppb.Timestamp)(nil), // 16: google.protobuf.Timestamp - (*fieldmaskpb.FieldMask)(nil), // 17: google.protobuf.FieldMask - (*emptypb.Empty)(nil), // 18: google.protobuf.Empty + (*UploadAttachmentRequest)(nil), // 10: memos.api.v1.UploadAttachmentRequest + (*UploadAttachmentSpec)(nil), // 11: memos.api.v1.UploadAttachmentSpec + (*UploadAttachmentResponse)(nil), // 12: memos.api.v1.UploadAttachmentResponse + (*ListAttachmentsRequest)(nil), // 13: memos.api.v1.ListAttachmentsRequest + (*ListAttachmentsResponse)(nil), // 14: memos.api.v1.ListAttachmentsResponse + (*GetAttachmentRequest)(nil), // 15: memos.api.v1.GetAttachmentRequest + (*UpdateAttachmentRequest)(nil), // 16: memos.api.v1.UpdateAttachmentRequest + (*DeleteAttachmentRequest)(nil), // 17: memos.api.v1.DeleteAttachmentRequest + (*BatchDeleteAttachmentsRequest)(nil), // 18: memos.api.v1.BatchDeleteAttachmentsRequest + (*timestamppb.Timestamp)(nil), // 19: google.protobuf.Timestamp + (*fieldmaskpb.FieldMask)(nil), // 20: google.protobuf.FieldMask + (*emptypb.Empty)(nil), // 21: google.protobuf.Empty } var file_api_v1_attachment_service_proto_depIdxs = []int32{ 0, // 0: memos.api.v1.MotionMedia.family:type_name -> memos.api.v1.MotionMediaFamily @@ -1262,30 +1545,35 @@ var file_api_v1_attachment_service_proto_depIdxs = []int32{ 7, // 3: memos.api.v1.MediaMetadata.video:type_name -> memos.api.v1.VideoMetadata 5, // 4: memos.api.v1.PhotoMetadata.capture_time:type_name -> memos.api.v1.MediaCaptureTime 6, // 5: memos.api.v1.PhotoMetadata.location:type_name -> memos.api.v1.MediaLocation - 16, // 6: memos.api.v1.Attachment.create_time:type_name -> google.protobuf.Timestamp + 19, // 6: memos.api.v1.Attachment.create_time:type_name -> google.protobuf.Timestamp 2, // 7: memos.api.v1.Attachment.motion_media:type_name -> memos.api.v1.MotionMedia 3, // 8: memos.api.v1.Attachment.media_metadata:type_name -> memos.api.v1.MediaMetadata 8, // 9: memos.api.v1.CreateAttachmentRequest.attachment:type_name -> memos.api.v1.Attachment - 8, // 10: memos.api.v1.ListAttachmentsResponse.attachments:type_name -> memos.api.v1.Attachment - 8, // 11: memos.api.v1.UpdateAttachmentRequest.attachment:type_name -> memos.api.v1.Attachment - 17, // 12: memos.api.v1.UpdateAttachmentRequest.update_mask:type_name -> google.protobuf.FieldMask - 9, // 13: memos.api.v1.AttachmentService.CreateAttachment:input_type -> memos.api.v1.CreateAttachmentRequest - 10, // 14: memos.api.v1.AttachmentService.ListAttachments:input_type -> memos.api.v1.ListAttachmentsRequest - 12, // 15: memos.api.v1.AttachmentService.GetAttachment:input_type -> memos.api.v1.GetAttachmentRequest - 13, // 16: memos.api.v1.AttachmentService.UpdateAttachment:input_type -> memos.api.v1.UpdateAttachmentRequest - 14, // 17: memos.api.v1.AttachmentService.DeleteAttachment:input_type -> memos.api.v1.DeleteAttachmentRequest - 15, // 18: memos.api.v1.AttachmentService.BatchDeleteAttachments:input_type -> memos.api.v1.BatchDeleteAttachmentsRequest - 8, // 19: memos.api.v1.AttachmentService.CreateAttachment:output_type -> memos.api.v1.Attachment - 11, // 20: memos.api.v1.AttachmentService.ListAttachments:output_type -> memos.api.v1.ListAttachmentsResponse - 8, // 21: memos.api.v1.AttachmentService.GetAttachment:output_type -> memos.api.v1.Attachment - 8, // 22: memos.api.v1.AttachmentService.UpdateAttachment:output_type -> memos.api.v1.Attachment - 18, // 23: memos.api.v1.AttachmentService.DeleteAttachment:output_type -> google.protobuf.Empty - 18, // 24: memos.api.v1.AttachmentService.BatchDeleteAttachments:output_type -> google.protobuf.Empty - 19, // [19:25] is the sub-list for method output_type - 13, // [13:19] is the sub-list for method input_type - 13, // [13:13] is the sub-list for extension type_name - 13, // [13:13] is the sub-list for extension extendee - 0, // [0:13] is the sub-list for field type_name + 11, // 10: memos.api.v1.UploadAttachmentRequest.spec:type_name -> memos.api.v1.UploadAttachmentSpec + 8, // 11: memos.api.v1.UploadAttachmentSpec.attachment:type_name -> memos.api.v1.Attachment + 8, // 12: memos.api.v1.UploadAttachmentResponse.attachment:type_name -> memos.api.v1.Attachment + 8, // 13: memos.api.v1.ListAttachmentsResponse.attachments:type_name -> memos.api.v1.Attachment + 8, // 14: memos.api.v1.UpdateAttachmentRequest.attachment:type_name -> memos.api.v1.Attachment + 20, // 15: memos.api.v1.UpdateAttachmentRequest.update_mask:type_name -> google.protobuf.FieldMask + 9, // 16: memos.api.v1.AttachmentService.CreateAttachment:input_type -> memos.api.v1.CreateAttachmentRequest + 10, // 17: memos.api.v1.AttachmentService.UploadAttachment:input_type -> memos.api.v1.UploadAttachmentRequest + 13, // 18: memos.api.v1.AttachmentService.ListAttachments:input_type -> memos.api.v1.ListAttachmentsRequest + 15, // 19: memos.api.v1.AttachmentService.GetAttachment:input_type -> memos.api.v1.GetAttachmentRequest + 16, // 20: memos.api.v1.AttachmentService.UpdateAttachment:input_type -> memos.api.v1.UpdateAttachmentRequest + 17, // 21: memos.api.v1.AttachmentService.DeleteAttachment:input_type -> memos.api.v1.DeleteAttachmentRequest + 18, // 22: memos.api.v1.AttachmentService.BatchDeleteAttachments:input_type -> memos.api.v1.BatchDeleteAttachmentsRequest + 8, // 23: memos.api.v1.AttachmentService.CreateAttachment:output_type -> memos.api.v1.Attachment + 12, // 24: memos.api.v1.AttachmentService.UploadAttachment:output_type -> memos.api.v1.UploadAttachmentResponse + 14, // 25: memos.api.v1.AttachmentService.ListAttachments:output_type -> memos.api.v1.ListAttachmentsResponse + 8, // 26: memos.api.v1.AttachmentService.GetAttachment:output_type -> memos.api.v1.Attachment + 8, // 27: memos.api.v1.AttachmentService.UpdateAttachment:output_type -> memos.api.v1.Attachment + 21, // 28: memos.api.v1.AttachmentService.DeleteAttachment:output_type -> google.protobuf.Empty + 21, // 29: memos.api.v1.AttachmentService.BatchDeleteAttachments:output_type -> google.protobuf.Empty + 23, // [23:30] is the sub-list for method output_type + 16, // [16:23] is the sub-list for method input_type + 16, // [16:16] is the sub-list for extension type_name + 16, // [16:16] is the sub-list for extension extendee + 0, // [0:16] is the sub-list for field type_name } func init() { file_api_v1_attachment_service_proto_init() } @@ -1302,13 +1590,17 @@ func file_api_v1_attachment_service_proto_init() { file_api_v1_attachment_service_proto_msgTypes[4].OneofWrappers = []any{} file_api_v1_attachment_service_proto_msgTypes[5].OneofWrappers = []any{} file_api_v1_attachment_service_proto_msgTypes[6].OneofWrappers = []any{} + file_api_v1_attachment_service_proto_msgTypes[8].OneofWrappers = []any{ + (*UploadAttachmentRequest_Spec)(nil), + (*UploadAttachmentRequest_UploadId)(nil), + } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_api_v1_attachment_service_proto_rawDesc), len(file_api_v1_attachment_service_proto_rawDesc)), NumEnums: 2, - NumMessages: 14, + NumMessages: 17, NumExtensions: 0, NumServices: 1, }, diff --git a/proto/gen/api/v1/attachment_service.pb.gw.go b/proto/gen/api/v1/attachment_service.pb.gw.go index f03ebd94..91afdaa7 100644 --- a/proto/gen/api/v1/attachment_service.pb.gw.go +++ b/proto/gen/api/v1/attachment_service.pb.gw.go @@ -76,6 +76,33 @@ func local_request_AttachmentService_CreateAttachment_0(ctx context.Context, mar return msg, metadata, err } +func request_AttachmentService_UploadAttachment_0(ctx context.Context, marshaler runtime.Marshaler, client AttachmentServiceClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { + var ( + protoReq UploadAttachmentRequest + metadata runtime.ServerMetadata + ) + if err := marshaler.NewDecoder(req.Body).Decode(&protoReq); err != nil && !errors.Is(err, io.EOF) { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) + } + if req.Body != nil { + _, _ = io.Copy(io.Discard, req.Body) + } + msg, err := client.UploadAttachment(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) + return msg, metadata, err +} + +func local_request_AttachmentService_UploadAttachment_0(ctx context.Context, marshaler runtime.Marshaler, server AttachmentServiceServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { + var ( + protoReq UploadAttachmentRequest + metadata runtime.ServerMetadata + ) + if err := marshaler.NewDecoder(req.Body).Decode(&protoReq); err != nil && !errors.Is(err, io.EOF) { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) + } + msg, err := server.UploadAttachment(ctx, &protoReq) + return msg, metadata, err +} + var filter_AttachmentService_ListAttachments_0 = &utilities.DoubleArray{Encoding: map[string]int{}, Base: []int(nil), Check: []int(nil)} func request_AttachmentService_ListAttachments_0(ctx context.Context, marshaler runtime.Marshaler, client AttachmentServiceClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { @@ -323,6 +350,26 @@ func RegisterAttachmentServiceHandlerServer(ctx context.Context, mux *runtime.Se } forward_AttachmentService_CreateAttachment_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) + mux.Handle(http.MethodPost, pattern_AttachmentService_UploadAttachment_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { + ctx, cancel := context.WithCancel(req.Context()) + defer cancel() + var stream runtime.ServerTransportStream + ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) + inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) + annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/memos.api.v1.AttachmentService/UploadAttachment", runtime.WithHTTPPathPattern("/api/v1/attachments:upload")) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + resp, md, err := local_request_AttachmentService_UploadAttachment_0(annotatedContext, inboundMarshaler, server, req, pathParams) + md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) + annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) + if err != nil { + runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) + return + } + forward_AttachmentService_UploadAttachment_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) + }) mux.Handle(http.MethodGet, pattern_AttachmentService_ListAttachments_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() @@ -480,6 +527,23 @@ func RegisterAttachmentServiceHandlerClient(ctx context.Context, mux *runtime.Se } forward_AttachmentService_CreateAttachment_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) + mux.Handle(http.MethodPost, pattern_AttachmentService_UploadAttachment_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { + ctx, cancel := context.WithCancel(req.Context()) + defer cancel() + inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) + annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/memos.api.v1.AttachmentService/UploadAttachment", runtime.WithHTTPPathPattern("/api/v1/attachments:upload")) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + resp, md, err := request_AttachmentService_UploadAttachment_0(annotatedContext, inboundMarshaler, client, req, pathParams) + annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) + if err != nil { + runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) + return + } + forward_AttachmentService_UploadAttachment_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) + }) mux.Handle(http.MethodGet, pattern_AttachmentService_ListAttachments_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() @@ -570,6 +634,7 @@ func RegisterAttachmentServiceHandlerClient(ctx context.Context, mux *runtime.Se var ( pattern_AttachmentService_CreateAttachment_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"api", "v1", "attachments"}, "")) + pattern_AttachmentService_UploadAttachment_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"api", "v1", "attachments"}, "upload")) pattern_AttachmentService_ListAttachments_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"api", "v1", "attachments"}, "")) pattern_AttachmentService_GetAttachment_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 1, 0, 4, 2, 5, 3}, []string{"api", "v1", "attachments", "name"}, "")) pattern_AttachmentService_UpdateAttachment_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 1, 0, 4, 2, 5, 3}, []string{"api", "v1", "attachments", "attachment.name"}, "")) @@ -579,6 +644,7 @@ var ( var ( forward_AttachmentService_CreateAttachment_0 = runtime.ForwardResponseMessage + forward_AttachmentService_UploadAttachment_0 = runtime.ForwardResponseMessage forward_AttachmentService_ListAttachments_0 = runtime.ForwardResponseMessage forward_AttachmentService_GetAttachment_0 = runtime.ForwardResponseMessage forward_AttachmentService_UpdateAttachment_0 = runtime.ForwardResponseMessage diff --git a/proto/gen/api/v1/attachment_service_grpc.pb.go b/proto/gen/api/v1/attachment_service_grpc.pb.go index bfeda059..69d15b0a 100644 --- a/proto/gen/api/v1/attachment_service_grpc.pb.go +++ b/proto/gen/api/v1/attachment_service_grpc.pb.go @@ -21,6 +21,7 @@ const _ = grpc.SupportPackageIsVersion9 const ( AttachmentService_CreateAttachment_FullMethodName = "/memos.api.v1.AttachmentService/CreateAttachment" + AttachmentService_UploadAttachment_FullMethodName = "/memos.api.v1.AttachmentService/UploadAttachment" AttachmentService_ListAttachments_FullMethodName = "/memos.api.v1.AttachmentService/ListAttachments" AttachmentService_GetAttachment_FullMethodName = "/memos.api.v1.AttachmentService/GetAttachment" AttachmentService_UpdateAttachment_FullMethodName = "/memos.api.v1.AttachmentService/UpdateAttachment" @@ -34,6 +35,11 @@ const ( type AttachmentServiceClient interface { // CreateAttachment creates a new attachment. CreateAttachment(ctx context.Context, in *CreateAttachmentRequest, opts ...grpc.CallOption) (*Attachment, error) + // UploadAttachment uploads a file in bounded chunks. The first call carries + // the spec and returns an upload_id; later calls carry that upload_id. + // Uploads are bound to the authenticated user, expire after 30 minutes of + // inactivity, and do not survive a server restart. + UploadAttachment(ctx context.Context, in *UploadAttachmentRequest, opts ...grpc.CallOption) (*UploadAttachmentResponse, error) // ListAttachments lists all attachments. ListAttachments(ctx context.Context, in *ListAttachmentsRequest, opts ...grpc.CallOption) (*ListAttachmentsResponse, error) // GetAttachment returns an attachment by name. @@ -64,6 +70,16 @@ func (c *attachmentServiceClient) CreateAttachment(ctx context.Context, in *Crea return out, nil } +func (c *attachmentServiceClient) UploadAttachment(ctx context.Context, in *UploadAttachmentRequest, opts ...grpc.CallOption) (*UploadAttachmentResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(UploadAttachmentResponse) + err := c.cc.Invoke(ctx, AttachmentService_UploadAttachment_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *attachmentServiceClient) ListAttachments(ctx context.Context, in *ListAttachmentsRequest, opts ...grpc.CallOption) (*ListAttachmentsResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(ListAttachmentsResponse) @@ -120,6 +136,11 @@ func (c *attachmentServiceClient) BatchDeleteAttachments(ctx context.Context, in type AttachmentServiceServer interface { // CreateAttachment creates a new attachment. CreateAttachment(context.Context, *CreateAttachmentRequest) (*Attachment, error) + // UploadAttachment uploads a file in bounded chunks. The first call carries + // the spec and returns an upload_id; later calls carry that upload_id. + // Uploads are bound to the authenticated user, expire after 30 minutes of + // inactivity, and do not survive a server restart. + UploadAttachment(context.Context, *UploadAttachmentRequest) (*UploadAttachmentResponse, error) // ListAttachments lists all attachments. ListAttachments(context.Context, *ListAttachmentsRequest) (*ListAttachmentsResponse, error) // GetAttachment returns an attachment by name. @@ -143,6 +164,9 @@ type UnimplementedAttachmentServiceServer struct{} func (UnimplementedAttachmentServiceServer) CreateAttachment(context.Context, *CreateAttachmentRequest) (*Attachment, error) { return nil, status.Error(codes.Unimplemented, "method CreateAttachment not implemented") } +func (UnimplementedAttachmentServiceServer) UploadAttachment(context.Context, *UploadAttachmentRequest) (*UploadAttachmentResponse, error) { + return nil, status.Error(codes.Unimplemented, "method UploadAttachment not implemented") +} func (UnimplementedAttachmentServiceServer) ListAttachments(context.Context, *ListAttachmentsRequest) (*ListAttachmentsResponse, error) { return nil, status.Error(codes.Unimplemented, "method ListAttachments not implemented") } @@ -197,6 +221,24 @@ func _AttachmentService_CreateAttachment_Handler(srv interface{}, ctx context.Co return interceptor(ctx, in, info, handler) } +func _AttachmentService_UploadAttachment_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(UploadAttachmentRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AttachmentServiceServer).UploadAttachment(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: AttachmentService_UploadAttachment_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AttachmentServiceServer).UploadAttachment(ctx, req.(*UploadAttachmentRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _AttachmentService_ListAttachments_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ListAttachmentsRequest) if err := dec(in); err != nil { @@ -298,6 +340,10 @@ var AttachmentService_ServiceDesc = grpc.ServiceDesc{ MethodName: "CreateAttachment", Handler: _AttachmentService_CreateAttachment_Handler, }, + { + MethodName: "UploadAttachment", + Handler: _AttachmentService_UploadAttachment_Handler, + }, { MethodName: "ListAttachments", Handler: _AttachmentService_ListAttachments_Handler, diff --git a/proto/gen/openapi.yaml b/proto/gen/openapi.yaml index 3368e245..95afba06 100644 --- a/proto/gen/openapi.yaml +++ b/proto/gen/openapi.yaml @@ -228,6 +228,35 @@ paths: application/json: schema: $ref: '#/components/schemas/Status' + /api/v1/attachments:upload: + post: + tags: + - AttachmentService + description: |- + UploadAttachment uploads a file in bounded chunks. The first call carries + the spec and returns an upload_id; later calls carry that upload_id. + Uploads are bound to the authenticated user, expire after 30 minutes of + inactivity, and do not survive a server restart. + operationId: AttachmentService_UploadAttachment + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/UploadAttachmentRequest' + required: true + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/UploadAttachmentResponse' + default: + description: Default error response + content: + application/json: + schema: + $ref: '#/components/schemas/Status' /api/v1/auth/me: get: tags: @@ -4573,6 +4602,76 @@ components: contentType: type: string description: Optional. The MIME type of the input audio. + UploadAttachmentRequest: + required: + - writeOffset + type: object + properties: + spec: + allOf: + - $ref: '#/components/schemas/UploadAttachmentSpec' + description: Starts a new upload. The same call may also carry data and finish_write. + uploadId: + type: string + description: Continues the upload identified by a previous response. + writeOffset: + type: string + description: |- + Required. Zero-based byte offset at which data is written. Must equal the + committed size, except when retrying the most recently accepted chunk + with identical bytes and offset, which is accepted without writing again. + data: + type: string + description: |- + Optional. File bytes, at most max_chunk_size long. With no data and + finish_write false, the call reports progress without writing, and + write_offset is ignored. + format: bytes + finishWrite: + type: boolean + description: |- + Optional. Finalize the upload after writing data. The committed size must + then equal total_size. Any later call for the same upload_id returns the + created attachment. + UploadAttachmentResponse: + type: object + properties: + uploadId: + type: string + description: Opaque ID for subsequent calls. This is not a resource name. + committedSize: + type: string + description: Number of file bytes committed so far. + attachment: + allOf: + - $ref: '#/components/schemas/Attachment' + description: Set once the upload has been finalized. + maxChunkSize: + type: integer + description: Maximum number of data bytes accepted in one call. + format: int32 + UploadAttachmentSpec: + required: + - attachment + type: object + properties: + attachment: + allOf: + - $ref: '#/components/schemas/Attachment' + description: |- + Required. Metadata for the attachment to create. content must be empty; + file bytes are sent in data. + attachmentId: + type: string + description: |- + Optional. The attachment ID to use for this attachment. + If empty, a unique ID will be generated. + Format: ^[a-zA-Z0-9]([a-zA-Z0-9-]{0,34}[a-zA-Z0-9])?$ + totalSize: + type: string + description: |- + Optional. Total size of the file in bytes before media processing. + Zero represents an empty file. UpsertMemoReactionRequest: required: - name diff --git a/server/router/api/v1/acl_config_test.go b/server/router/api/v1/acl_config_test.go index 633b0224..b2efa6ec 100644 --- a/server/router/api/v1/acl_config_test.go +++ b/server/router/api/v1/acl_config_test.go @@ -48,6 +48,7 @@ func TestPublicMethodsArePublic(t *testing.T) { // TestProtectedMethodsRequireAuth verifies that non-public methods are recognized as protected. func TestProtectedMethodsRequireAuth(t *testing.T) { protectedMethods := []string{ + "/memos.api.v1.AttachmentService/UploadAttachment", // Auth Service - logout and get current user require auth "/memos.api.v1.AuthService/SignOut", "/memos.api.v1.AuthService/GetCurrentUser", diff --git a/server/router/api/v1/attachment_exif_test.go b/server/router/api/v1/attachment_exif_test.go index 51cc6137..9531a1a0 100644 --- a/server/router/api/v1/attachment_exif_test.go +++ b/server/router/api/v1/attachment_exif_test.go @@ -104,7 +104,7 @@ func TestStripImageExif(t *testing.T) { t.Run("strip JPEG metadata", func(t *testing.T) { t.Parallel() - strippedData, err := stripImageExif(originalData, "image/jpeg") + strippedData, err := stripImageExif(bytes.NewReader(originalData), "image/jpeg") require.NoError(t, err) assert.NotEmpty(t, strippedData) @@ -118,7 +118,7 @@ func TestStripImageExif(t *testing.T) { t.Run("strip JPG metadata (alternate extension)", func(t *testing.T) { t.Parallel() - strippedData, err := stripImageExif(originalData, "image/jpg") + strippedData, err := stripImageExif(bytes.NewReader(originalData), "image/jpg") require.NoError(t, err) assert.NotEmpty(t, strippedData) @@ -136,7 +136,7 @@ func TestStripImageExif(t *testing.T) { err := imaging.Encode(&pngBuf, img, imaging.PNG) require.NoError(t, err) - strippedData, err := stripImageExif(pngBuf.Bytes(), "image/png") + strippedData, err := stripImageExif(bytes.NewReader(pngBuf.Bytes()), "image/png") require.NoError(t, err) assert.NotEmpty(t, strippedData) @@ -151,7 +151,7 @@ func TestStripImageExif(t *testing.T) { t.Parallel() // WebP format will be converted to JPEG - strippedData, err := stripImageExif(originalData, "image/webp") + strippedData, err := stripImageExif(bytes.NewReader(originalData), "image/webp") require.NoError(t, err) assert.NotEmpty(t, strippedData) @@ -164,7 +164,7 @@ func TestStripImageExif(t *testing.T) { t.Run("handle HEIC format by converting to JPEG", func(t *testing.T) { t.Parallel() - strippedData, err := stripImageExif(originalData, "image/heic") + strippedData, err := stripImageExif(bytes.NewReader(originalData), "image/heic") require.NoError(t, err) assert.NotEmpty(t, strippedData) @@ -178,7 +178,7 @@ func TestStripImageExif(t *testing.T) { t.Parallel() invalidData := []byte("not an image") - _, err := stripImageExif(invalidData, "image/jpeg") + _, err := stripImageExif(bytes.NewReader(invalidData), "image/jpeg") assert.Error(t, err) assert.Contains(t, err.Error(), "failed to decode image") }) @@ -187,7 +187,7 @@ func TestStripImageExif(t *testing.T) { t.Parallel() emptyData := []byte{} - _, err := stripImageExif(emptyData, "image/jpeg") + _, err := stripImageExif(bytes.NewReader(emptyData), "image/jpeg") assert.Error(t, err) }) } @@ -195,7 +195,7 @@ func TestStripImageExif(t *testing.T) { func TestValidateImagePixelCountRejectsOversizedDimensions(t *testing.T) { t.Parallel() - err := validateImagePixelCount(testPNGHeaderWithDimensions(100_000, 100_000)) + err := validateImageReaderPixelCount(bytes.NewReader(testPNGHeaderWithDimensions(100_000, 100_000))) require.Error(t, err) require.Contains(t, err.Error(), "image dimensions exceed maximum") } @@ -203,7 +203,7 @@ func TestValidateImagePixelCountRejectsOversizedDimensions(t *testing.T) { func TestStripImageExifRejectsOversizedDimensionsBeforeDecode(t *testing.T) { t.Parallel() - _, err := stripImageExif(testPNGHeaderWithDimensions(100_000, 100_000), "image/png") + _, err := stripImageExif(bytes.NewReader(testPNGHeaderWithDimensions(100_000, 100_000)), "image/png") require.Error(t, err) require.Contains(t, err.Error(), "image dimensions exceed maximum") } diff --git a/server/router/api/v1/attachment_service.go b/server/router/api/v1/attachment_service.go index 07967b36..b22ab0c1 100644 --- a/server/router/api/v1/attachment_service.go +++ b/server/router/api/v1/attachment_service.go @@ -1,10 +1,12 @@ package v1 import ( + "bytes" "context" - "encoding/binary" "fmt" + "io" "log/slog" + "math" "mime" "net/http" "path/filepath" @@ -15,16 +17,16 @@ import ( "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/emptypb" + "github.com/usememos/memos/internal/motionphoto" v1pb "github.com/usememos/memos/proto/gen/api/v1" + storepb "github.com/usememos/memos/proto/gen/store" "github.com/usememos/memos/store" ) const ( - // The upload memory buffer is 32 MiB. - // It should be kept low, so RAM usage doesn't get out of control. - // This is unrelated to maximum upload size limit, which is now set through system setting. - MaxUploadBufferSizeBytes = 32 << 20 - MebiByte = 1024 * 1024 + // DefaultUploadSizeLimitBytes applies when no upload size limit is configured. + DefaultUploadSizeLimitBytes = 32 << 20 + MebiByte = 1024 * 1024 // defaultJPEGQuality is the JPEG quality used when re-encoding images for EXIF stripping. // Quality 95 maintains visual quality while ensuring metadata is removed. @@ -71,7 +73,7 @@ func detectAttachmentMimeType(filename string, content []byte) string { return http.DetectContentType(content) } -func (s *APIV1Service) CreateAttachment(ctx context.Context, request *v1pb.CreateAttachmentRequest) (*v1pb.Attachment, error) { +func (s *APIV1Service) prepareAttachment(ctx context.Context, request *v1pb.CreateAttachmentRequest) (*store.Attachment, error) { user, err := s.fetchCurrentUser(ctx) if err != nil { return nil, status.Errorf(codes.Internal, "failed to get current user: %v", err) @@ -135,21 +137,6 @@ func (s *APIV1Service) CreateAttachment(ctx context.Context, request *v1pb.Creat create.Payload.MediaMetadata = inputMediaMetadata } - instanceStorageSetting, err := s.Store.GetInstanceStorageSetting(ctx) - if err != nil { - return nil, status.Errorf(codes.Internal, "failed to get instance storage setting: %v", err) - } - size := binary.Size(request.Attachment.Content) - uploadSizeLimit := int(instanceStorageSetting.UploadSizeLimitMb) * MebiByte - if uploadSizeLimit == 0 { - uploadSizeLimit = MaxUploadBufferSizeBytes - } - if size > uploadSizeLimit { - return nil, status.Errorf(codes.InvalidArgument, "file size exceeds the limit") - } - create.Size = int64(size) - create.Blob = request.Attachment.Content - if request.Attachment.Memo != nil { memoUID, err := ExtractMemoUIDFromName(*request.Attachment.Memo) if err != nil { @@ -172,21 +159,77 @@ func (s *APIV1Service) CreateAttachment(ctx context.Context, request *v1pb.Creat create.Policy = memoWritePolicy(user.ID, false) } - if create.Payload == nil || create.Payload.MotionMedia == nil { - if detectedMotion := detectAndroidMotionMedia(create.Blob, create.Type, attachmentUID); detectedMotion != nil { + return create, nil +} + +func (s *APIV1Service) CreateAttachment(ctx context.Context, request *v1pb.CreateAttachmentRequest) (*v1pb.Attachment, error) { + create, err := s.prepareAttachment(ctx, request) + if err != nil { + return nil, err + } + instanceStorageSetting, err := s.Store.GetInstanceStorageSetting(ctx) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to get instance storage setting: %v", err) + } + content := request.Attachment.Content + if err := checkUploadSize(instanceStorageSetting, int64(len(content))); err != nil { + return nil, err + } + create.Size = int64(len(content)) + return s.processAndSaveAttachment(ctx, create, instanceStorageSetting, bytes.NewReader(content)) +} + +func attachmentUploadLimit(setting *storepb.InstanceStorageSetting) int64 { + if setting.UploadSizeLimitMb <= 0 { + return DefaultUploadSizeLimitBytes + } + return min(setting.UploadSizeLimitMb, math.MaxInt64/MebiByte) * MebiByte +} + +func checkUploadSize(setting *storepb.InstanceStorageSetting, size int64) error { + if size > attachmentUploadLimit(setting) { + return status.Errorf(codes.ResourceExhausted, "file size exceeds the limit") + } + return nil +} + +// attachmentSource is the file content handed to the processing pipeline: a +// bytes.Reader for the one-shot RPC, an *os.File for a chunked upload. +type attachmentSource interface { + io.ReadSeeker + io.ReaderAt +} + +// processAndSaveAttachment detects motion photos, strips EXIF metadata, stores +// the content, and creates the database row. create.Size must hold the source +// length on entry; it is updated when stripping re-encodes the image. +func (s *APIV1Service) processAndSaveAttachment(ctx context.Context, create *store.Attachment, instanceStorageSetting *storepb.InstanceStorageSetting, source attachmentSource) (*v1pb.Attachment, error) { + if create.Payload.GetMotionMedia() == nil && (create.Type == "image/jpeg" || create.Type == "image/jpg") { + detected, err := motionphoto.DetectJPEGReader(source, create.Size) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to inspect motion photo: %v", err) + } + if detected != nil { create.Payload = ensureAttachmentPayload(create.Payload) - create.Payload.MotionMedia = detectedMotion + create.Payload.MotionMedia = &storepb.MotionMedia{ + Family: storepb.MotionMediaFamily_ANDROID_MOTION_PHOTO, + Role: storepb.MotionMediaRole_CONTAINER, + GroupId: create.UID, + PresentationTimestampUs: detected.PresentationTimestampUs, + HasEmbeddedVideo: true, + } } } - // Strip EXIF metadata from images for privacy protection. - // This removes sensitive information like GPS location, device details, etc. + content := io.ReadSeeker(source) + // Strip EXIF metadata from images for privacy protection. Motion photo + // containers are kept intact because re-encoding would drop the video. if shouldStripExif(create.Type) && !isAndroidMotionContainer(create.Payload.GetMotionMedia()) { release, err := s.acquireImageProcessingSlot(ctx) if err != nil { return nil, status.Errorf(codes.ResourceExhausted, "too many image processing requests") } - strippedBlob, stripErr := stripImageExif(create.Blob, create.Type) + stripped, stripErr := stripImageExif(source, create.Type) release() if stripErr != nil { // Log warning but continue with original image to ensure uploads don't fail. @@ -195,15 +238,21 @@ func (s *APIV1Service) CreateAttachment(ctx context.Context, request *v1pb.Creat slog.String("filename", create.Filename), slog.String("error", stripErr.Error())) } else { - create.Blob = strippedBlob - create.Size = int64(len(strippedBlob)) + content = bytes.NewReader(stripped) + create.Size = int64(len(stripped)) } } - if err := saveAttachmentBlobWithInstanceStorageSetting(ctx, s.Profile, s.Store, create, instanceStorageSetting); err != nil { + if _, err := content.Seek(0, io.SeekStart); err != nil { + return nil, status.Errorf(codes.Internal, "failed to rewind attachment content: %v", err) + } + if err := saveAttachmentContent(ctx, s.Profile, s.Store, create, instanceStorageSetting, content); err != nil { return nil, status.Errorf(codes.Internal, "failed to save attachment blob: %v", err) } + return s.persistAttachment(ctx, create, instanceStorageSetting) +} +func (s *APIV1Service) persistAttachment(ctx context.Context, create *store.Attachment, instanceStorageSetting *storepb.InstanceStorageSetting) (*v1pb.Attachment, error) { attachment, err := s.Store.CreateAttachment(ctx, create) if err != nil { createErr := mapMemoWriteError(err, "failed to create attachment") diff --git a/server/router/api/v1/attachment_service_image.go b/server/router/api/v1/attachment_service_image.go index 4db0c446..dc15965f 100644 --- a/server/router/api/v1/attachment_service_image.go +++ b/server/router/api/v1/attachment_service_image.go @@ -4,13 +4,13 @@ import ( "bytes" "context" "image" + "io" "github.com/disintegration/imaging" "github.com/pkg/errors" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/usememos/memos/internal/motionphoto" v1pb "github.com/usememos/memos/proto/gen/api/v1" storepb "github.com/usememos/memos/proto/gen/store" ) @@ -38,25 +38,6 @@ func validateClientMotionMedia(motion *v1pb.MotionMedia, attachmentUID string) ( return storeMotion, nil } -func detectAndroidMotionMedia(blob []byte, mimeType, attachmentUID string) *storepb.MotionMedia { - if mimeType != "image/jpeg" && mimeType != "image/jpg" { - return nil - } - - detection := motionphoto.DetectJPEG(blob) - if detection == nil { - return nil - } - - return &storepb.MotionMedia{ - Family: storepb.MotionMediaFamily_ANDROID_MOTION_PHOTO, - Role: storepb.MotionMediaRole_CONTAINER, - GroupId: attachmentUID, - PresentationTimestampUs: detection.PresentationTimestampUs, - HasEmbeddedVideo: true, - } -} - // shouldStripExif checks if the MIME type is an image format that may contain EXIF metadata. // Returns true for formats like JPEG, TIFF, WebP, HEIC, and HEIF which commonly contain // privacy-sensitive metadata such as GPS coordinates, camera settings, and device information. @@ -76,8 +57,8 @@ func (s *APIV1Service) acquireImageProcessingSlot(ctx context.Context) (func(), }, nil } -func validateImagePixelCount(imageData []byte) error { - config, _, err := image.DecodeConfig(bytes.NewReader(imageData)) +func validateImageReaderPixelCount(reader io.Reader) error { + config, _, err := image.DecodeConfig(reader) if err != nil { // Some formats supported by imaging do not expose dimensions through // the standard image registry. Let the full decoder handle those. @@ -97,7 +78,8 @@ func validateImagePixelCount(imageData []byte) error { // // The function preserves the correct image orientation by applying EXIF orientation tags // during decoding before stripping all metadata. Images are re-encoded with high quality -// to minimize visual degradation. +// to minimize visual degradation. The re-encoded output is returned in memory; its size +// is bounded by maxImagePixels, which the decoder already has to hold. // // Supported formats: // - JPEG/JPG: Re-encoded as JPEG with quality 95 @@ -105,34 +87,28 @@ func validateImagePixelCount(imageData []byte) error { // - TIFF/WebP/HEIC/HEIF: Re-encoded as JPEG with quality 95 // // Returns the cleaned image data without any EXIF metadata, or an error if processing fails. -func stripImageExif(imageData []byte, mimeType string) ([]byte, error) { - if err := validateImagePixelCount(imageData); err != nil { +func stripImageExif(source io.ReadSeeker, mimeType string) ([]byte, error) { + if _, err := source.Seek(0, io.SeekStart); err != nil { + return nil, errors.Wrap(err, "failed to rewind image") + } + if err := validateImageReaderPixelCount(source); err != nil { return nil, err } - - // Decode image with automatic EXIF orientation correction. - // This ensures the image displays correctly after metadata removal. - img, err := imaging.Decode(bytes.NewReader(imageData), imaging.AutoOrientation(true)) + if _, err := source.Seek(0, io.SeekStart); err != nil { + return nil, errors.Wrap(err, "failed to rewind image") + } + img, err := imaging.Decode(source, imaging.AutoOrientation(true)) if err != nil { return nil, errors.Wrap(err, "failed to decode image") } - - // Re-encode the image without EXIF metadata. var buf bytes.Buffer - var encodeErr error - if mimeType == "image/png" { - // Preserve PNG format for lossless encoding - encodeErr = imaging.Encode(&buf, img, imaging.PNG) + err = imaging.Encode(&buf, img, imaging.PNG) } else { - // For JPEG, TIFF, WebP, HEIC, HEIF - re-encode as JPEG. - // This ensures EXIF is stripped and provides good compression. - encodeErr = imaging.Encode(&buf, img, imaging.JPEG, imaging.JPEGQuality(defaultJPEGQuality)) + err = imaging.Encode(&buf, img, imaging.JPEG, imaging.JPEGQuality(defaultJPEGQuality)) } - - if encodeErr != nil { - return nil, errors.Wrap(encodeErr, "failed to encode image") + if err != nil { + return nil, errors.Wrap(err, "failed to encode image") } - return buf.Bytes(), nil } diff --git a/server/router/api/v1/attachment_service_storage.go b/server/router/api/v1/attachment_service_storage.go index 0aa41fa9..dcbfc4c9 100644 --- a/server/router/api/v1/attachment_service_storage.go +++ b/server/router/api/v1/attachment_service_storage.go @@ -1,7 +1,6 @@ package v1 import ( - "bytes" "context" "fmt" "io" @@ -46,21 +45,29 @@ func convertAttachmentFromStore(attachment *store.Attachment) *v1pb.Attachment { return attachmentMessage } -// SaveAttachmentBlob saves the blob of attachment based on the storage config. -func SaveAttachmentBlob(ctx context.Context, profile *profile.Profile, stores *store.Store, create *store.Attachment) error { - instanceStorageSetting, err := stores.GetInstanceStorageSetting(ctx) - if err != nil { - return errors.Wrap(err, "Failed to find instance storage setting") - } - return saveAttachmentBlobWithInstanceStorageSetting(ctx, profile, stores, create, instanceStorageSetting) +// attachmentContextReader stops a long local copy once the request context is +// canceled; io.Copy has no context of its own, unlike the S3 client. +type attachmentContextReader struct { + ctx context.Context + reader io.Reader } -func saveAttachmentBlobWithInstanceStorageSetting( +func (r *attachmentContextReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.reader.Read(p) +} + +// saveAttachmentContent writes content to the default storage and records +// where it went on create. Database storage keeps the bytes on create.Blob. +func saveAttachmentContent( ctx context.Context, profile *profile.Profile, stores *store.Store, create *store.Attachment, instanceStorageSetting *storepb.InstanceStorageSetting, + content io.Reader, ) error { defaultStorage := store.GetDefaultStorage(instanceStorageSetting) if defaultStorage == nil { @@ -99,9 +106,24 @@ func saveAttachmentBlobWithInstanceStorageSetting( return errors.Wrap(err, "Failed to create directory") } - // Write the blob to the file. - if err := os.WriteFile(osPath, create.Blob, 0644); err != nil { - return errors.Wrap(err, "Failed to write file") + // Stage in a temp file so partial content never appears at the final path. + file, err := os.CreateTemp(dir, ".memos-upload-*") + if err != nil { + return errors.Wrap(err, "failed to create attachment file") + } + defer os.Remove(file.Name()) + defer file.Close() + if _, err := io.Copy(file, &attachmentContextReader{ctx: ctx, reader: content}); err != nil { + return errors.Wrap(err, "failed to write attachment file") + } + if err := file.Chmod(0644); err != nil { + return errors.Wrap(err, "failed to set attachment permissions") + } + if err := file.Close(); err != nil { + return errors.Wrap(err, "failed to close attachment file") + } + if err := os.Rename(file.Name(), osPath); err != nil { + return errors.Wrap(err, "failed to finalize attachment file") } create.Reference = internalPath create.Blob = nil @@ -117,7 +139,7 @@ func saveAttachmentBlobWithInstanceStorageSetting( filepathTemplate = filepath.Join(filepathTemplate, "{filename}") } filepathTemplate = replaceFilenameWithPathTemplate(filepathTemplate, create.Filename) - key, err := driver.UploadObject(ctx, filepathTemplate, create.Type, bytes.NewReader(create.Blob)) + key, err := driver.UploadObject(ctx, filepathTemplate, create.Type, content) if err != nil { return errors.Wrap(err, "failed to upload via storage driver") } @@ -133,6 +155,12 @@ func saveAttachmentBlobWithInstanceStorageSetting( }, } create.Payload = payload + } else { + blob, err := io.ReadAll(content) + if err != nil { + return errors.Wrap(err, "failed to read attachment content") + } + create.Blob = blob } return nil diff --git a/server/router/api/v1/attachment_upload.go b/server/router/api/v1/attachment_upload.go new file mode 100644 index 00000000..2cbd1036 --- /dev/null +++ b/server/router/api/v1/attachment_upload.go @@ -0,0 +1,214 @@ +package v1 + +import ( + "context" + "crypto/sha256" + "io" + "os" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + + v1pb "github.com/usememos/memos/proto/gen/api/v1" + "github.com/usememos/memos/store" +) + +// UploadAttachment accepts bounded unary chunks. A call carrying a spec opens +// a new upload; a call carrying an upload ID continues one. Either kind may +// write data and finalize. +func (s *APIV1Service) UploadAttachment(ctx context.Context, request *v1pb.UploadAttachmentRequest) (*v1pb.UploadAttachmentResponse, error) { + user, err := s.requireCurrentSpaceUser(ctx) + if err != nil { + return nil, err + } + if len(request.Data) > attachmentUploadChunkSize { + return nil, status.Errorf(codes.ResourceExhausted, "upload chunk exceeds the limit") + } + var id string + var upload *attachmentUpload + switch u := request.Upload.(type) { + case *v1pb.UploadAttachmentRequest_Spec: + id, upload, err = s.startAttachmentUpload(ctx, request, u.Spec, user.ID) + if err != nil { + return nil, err + } + upload.mu.Lock() + case *v1pb.UploadAttachmentRequest_UploadId: + id = u.UploadId + upload, err = s.attachmentUploads.get(id, user.ID) + if err != nil { + return nil, err + } + upload.mu.Lock() + if !time.Now().Before(upload.expireTime) { + upload.mu.Unlock() + return nil, status.Errorf(codes.NotFound, "upload not found or expired") + } + default: + return nil, status.Errorf(codes.InvalidArgument, "spec or upload_id is required") + } + defer upload.mu.Unlock() + if err := ctx.Err(); err != nil { + return nil, status.FromContextError(err).Err() + } + if err := upload.write(ctx, request); err != nil { + return nil, err + } + var attachment *v1pb.Attachment + if upload.complete { + // Recheck access and existence; a completed upload must not resurrect a + // deleted attachment or disclose metadata after access is revoked. + attachment, err = s.GetAttachment(ctx, &v1pb.GetAttachmentRequest{Name: "attachments/" + upload.uid}) + } else if request.FinishWrite { + if upload.committedSize != upload.totalSize { + return nil, status.Errorf(codes.FailedPrecondition, "upload is incomplete") + } + attachment, err = s.finishAttachmentUpload(ctx, upload) + if err == nil { + upload.complete = true + os.Remove(upload.path) + } + } + if err != nil { + return nil, err + } + upload.expireTime = time.Now().Add(attachmentUploadTTL) + return &v1pb.UploadAttachmentResponse{ + UploadId: id, CommittedSize: upload.committedSize, Attachment: attachment, MaxChunkSize: attachmentUploadChunkSize, + }, nil +} + +// startAttachmentUpload validates the spec and registers a new upload. Every +// check that can fail on the accompanying data runs first, so a rejected call +// never leaves an orphaned upload behind. +func (s *APIV1Service) startAttachmentUpload(ctx context.Context, request *v1pb.UploadAttachmentRequest, spec *v1pb.UploadAttachmentSpec, ownerID int32) (string, *attachmentUpload, error) { + if spec.Attachment == nil { + return "", nil, status.Errorf(codes.InvalidArgument, "spec.attachment is required") + } + if len(spec.Attachment.Content) != 0 { + return "", nil, status.Errorf(codes.InvalidArgument, "spec.attachment.content must be empty; send file bytes in data") + } + if proto.Size(spec) > attachmentUploadMetadataLimit || spec.TotalSize < 0 { + return "", nil, status.Errorf(codes.InvalidArgument, "invalid attachment metadata or total_size") + } + if request.WriteOffset != 0 { + return "", nil, status.Errorf(codes.OutOfRange, "write_offset must be 0 for a new upload") + } + if int64(len(request.Data)) > spec.TotalSize { + return "", nil, status.Errorf(codes.InvalidArgument, "data exceeds total_size") + } + setting, err := s.Store.GetInstanceStorageSetting(ctx) + if err != nil { + return "", nil, status.Errorf(codes.Internal, "failed to get storage setting: %v", err) + } + if err := checkUploadSize(setting, spec.TotalSize); err != nil { + return "", nil, err + } + // Validate before allocating a temporary file. Keep the original MIME type + // so an omitted type can be sniffed from real bytes at finalization. + metadata := proto.CloneOf(spec.Attachment) + validationMetadata := proto.CloneOf(metadata) + if validationMetadata.Type == "" { + // Type-dependent metadata validation must wait for content sniffing. + validationMetadata.MediaMetadata = nil + } + create, err := s.prepareAttachment(ctx, &v1pb.CreateAttachmentRequest{Attachment: validationMetadata, AttachmentId: spec.AttachmentId}) + if err != nil { + return "", nil, err + } + upload := &attachmentUpload{ownerID: ownerID, metadata: metadata, uid: create.UID, totalSize: spec.TotalSize} + id, err := s.attachmentUploads.create(s.Profile.Data, upload) + if err != nil { + return "", nil, err + } + return id, upload, nil +} + +func (u *attachmentUpload) write(ctx context.Context, request *v1pb.UploadAttachmentRequest) error { + if len(request.Data) == 0 { + if request.FinishWrite && request.WriteOffset != u.committedSize { + return status.Errorf(codes.OutOfRange, "write_offset must equal committed_size") + } + return nil + } + digest := sha256.Sum256(request.Data) + if u.committedSize > 0 && request.WriteOffset+int64(len(request.Data)) == u.committedSize && digest == u.lastDigest { + return nil // A lost response can be retried without appending bytes twice. + } + if u.complete { + return status.Errorf(codes.FailedPrecondition, "upload is already complete") + } + if request.WriteOffset != u.committedSize { + return status.Errorf(codes.OutOfRange, "write_offset must equal committed_size") + } + if int64(len(request.Data)) > u.totalSize-u.committedSize { + return status.Errorf(codes.InvalidArgument, "data exceeds total_size") + } + file, err := os.OpenFile(u.path, os.O_WRONLY, 0600) + if err != nil { + return status.Errorf(codes.Internal, "failed to open upload file: %v", err) + } + defer file.Close() + n, err := file.WriteAt(request.Data, u.committedSize) + if err == nil && n != len(request.Data) { + err = io.ErrShortWrite + } + if err == nil { + err = ctx.Err() + } + if err != nil { + if truncateErr := file.Truncate(u.committedSize); truncateErr != nil { + // The file no longer matches committedSize; revoke the upload. + u.expireTime = time.Time{} + os.Remove(u.path) + } + return status.Errorf(codes.Internal, "failed to write upload chunk: %v", err) + } + u.lastDigest = digest + u.committedSize += int64(n) + return nil +} + +func (s *APIV1Service) finishAttachmentUpload(ctx context.Context, upload *attachmentUpload) (*v1pb.Attachment, error) { + if upload.finalizeAttempted { + // A database operation may commit before returning an error. Resolve that + // outcome before saving another object under the upload's stable UID. + persisted, err := s.Store.GetAttachment(ctx, &store.FindAttachment{UID: &upload.uid}) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to resolve previous finalization: %v", err) + } + if persisted != nil { + if persisted.CreatorID != upload.ownerID { + return nil, status.Errorf(codes.AlreadyExists, "attachment ID already exists") + } + return s.GetAttachment(ctx, &v1pb.GetAttachmentRequest{Name: "attachments/" + upload.uid}) + } + } + setting, err := s.Store.GetInstanceStorageSetting(ctx) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to get storage setting: %v", err) + } + if err := checkUploadSize(setting, upload.totalSize); err != nil { + return nil, err + } + file, err := os.Open(upload.path) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to open upload file: %v", err) + } + defer file.Close() + metadata := proto.CloneOf(upload.metadata) + metadata.Content = make([]byte, min(upload.totalSize, 512)) + if _, err := io.ReadFull(file, metadata.Content); err != nil { + return nil, status.Errorf(codes.Internal, "failed to read attachment header: %v", err) + } + // Revalidate memo permissions and media metadata with the actual MIME type. + create, err := s.prepareAttachment(ctx, &v1pb.CreateAttachmentRequest{Attachment: metadata, AttachmentId: upload.uid}) + if err != nil { + return nil, err + } + create.Size = upload.totalSize + upload.finalizeAttempted = true + return s.processAndSaveAttachment(ctx, create, setting, file) +} diff --git a/server/router/api/v1/attachment_upload_state.go b/server/router/api/v1/attachment_upload_state.go new file mode 100644 index 00000000..45e7ddf3 --- /dev/null +++ b/server/router/api/v1/attachment_upload_state.go @@ -0,0 +1,194 @@ +package v1 + +import ( + "crypto/rand" + "crypto/sha256" + "os" + "path/filepath" + "sync" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + v1pb "github.com/usememos/memos/proto/gen/api/v1" +) + +const ( + attachmentUploadChunkSize = 2 << 20 + attachmentUploadRequestLimit = 4 << 20 + attachmentUploadMetadataLimit = 64 << 10 + attachmentUploadTTL = 30 * time.Minute + attachmentUploadMaxSessions = 1024 + attachmentUploadMaxActive = 128 + attachmentUploadMaxActivePerUser = 8 + attachmentUploadTempPrefix = ".memos-rpc-upload-" + attachmentUploadProcedure = "/memos.api.v1.AttachmentService/UploadAttachment" +) + +// Upload state is private to the process; only Attachment is an API resource. +// Each upload holds metadata and the last chunk's digest, never its contents. +type attachmentUpload struct { + mu sync.Mutex + ownerID int32 + metadata *v1pb.Attachment + uid string + path string + totalSize int64 + committedSize int64 + // expireTime is zeroed to revoke an upload before its TTL elapses. + expireTime time.Time + // lastDigest identifies the most recently accepted chunk so a client can + // safely resend it after losing the response. + lastDigest [sha256.Size]byte + finalizeAttempted bool + complete bool +} + +type attachmentUploads struct { + mu sync.Mutex + entries map[string]*attachmentUpload + stop chan struct{} + done chan struct{} + closed bool +} + +func (m *attachmentUploads) startLocked(dir string) { + if m.entries != nil { + return + } + m.entries = make(map[string]*attachmentUpload) + m.stop = make(chan struct{}) + m.done = make(chan struct{}) + go func() { + defer close(m.done) + ticker := time.NewTicker(time.Minute) + defer ticker.Stop() + for { + select { + case <-m.stop: + return + case now := <-ticker.C: + m.mu.Lock() + m.sweepLocked(now, 0) + m.removeOrphansLocked(dir, now) + m.mu.Unlock() + } + } + }() +} + +// sweepLocked drops expired uploads and counts the unfinished ones, both in +// total and for ownerID. An upload locked by an in-flight request is +// unfinished by definition. +func (m *attachmentUploads) sweepLocked(now time.Time, ownerID int32) (owned, total int) { + count := func(upload *attachmentUpload) { + total++ + if upload.ownerID == ownerID { + owned++ + } + } + for id, upload := range m.entries { + // Never hold the manager lock while waiting for a file write or finalize. + if !upload.mu.TryLock() { + count(upload) + continue + } + if !now.Before(upload.expireTime) { + os.Remove(upload.path) + delete(m.entries, id) + } else if !upload.complete { + count(upload) + } + upload.mu.Unlock() + } + return owned, total +} + +func (m *attachmentUploads) removeOrphansLocked(dir string, now time.Time) { + tracked := make(map[string]bool, len(m.entries)) + for _, upload := range m.entries { + tracked[upload.path] = true // path is immutable after insertion. + } + paths, _ := filepath.Glob(filepath.Join(dir, attachmentUploadTempPrefix+"*")) + for _, path := range paths { + if tracked[path] { + continue + } + if info, err := os.Stat(path); err == nil && info.Mode().IsRegular() && now.Sub(info.ModTime()) >= attachmentUploadTTL { + os.Remove(path) + } + } +} + +func (m *attachmentUploads) create(dir string, upload *attachmentUpload) (string, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.closed { + return "", status.Errorf(codes.Unavailable, "server is shutting down") + } + m.startLocked(dir) + owned, total := m.sweepLocked(time.Now(), upload.ownerID) + if owned >= attachmentUploadMaxActivePerUser { + return "", status.Errorf(codes.ResourceExhausted, "too many active uploads") + } + if len(m.entries) >= attachmentUploadMaxSessions || total >= attachmentUploadMaxActive { + return "", status.Errorf(codes.ResourceExhausted, "too many uploads") + } + file, err := os.CreateTemp(dir, attachmentUploadTempPrefix+"*") + if err != nil { + return "", status.Errorf(codes.Internal, "failed to create upload file: %v", err) + } + if err := file.Close(); err != nil { + os.Remove(file.Name()) + return "", status.Errorf(codes.Internal, "failed to close upload file: %v", err) + } + upload.path = file.Name() + upload.expireTime = time.Now().Add(attachmentUploadTTL) + id := rand.Text() + m.entries[id] = upload + return id, nil +} + +func (m *attachmentUploads) get(id string, ownerID int32) (*attachmentUpload, error) { + m.mu.Lock() + upload := m.entries[id] + closed := m.closed + m.mu.Unlock() + if closed { + return nil, status.Errorf(codes.Unavailable, "server is shutting down") + } + if upload == nil || upload.ownerID != ownerID { + return nil, status.Errorf(codes.NotFound, "upload not found or expired") + } + return upload, nil +} + +// CloseAttachmentUploads stops expiration work and removes pending upload files. +// Call it after draining HTTP requests during server shutdown. +func (s *APIV1Service) CloseAttachmentUploads() { + m := &s.attachmentUploads + m.mu.Lock() + if m.closed { + m.mu.Unlock() + return + } + m.closed = true + if m.stop != nil { + close(m.stop) + } + done := m.done + m.mu.Unlock() + if done != nil { + <-done + } + m.mu.Lock() + defer m.mu.Unlock() + for id, upload := range m.entries { + upload.mu.Lock() + upload.expireTime = time.Time{} + os.Remove(upload.path) + upload.mu.Unlock() + delete(m.entries, id) + } +} diff --git a/server/router/api/v1/attachment_upload_test.go b/server/router/api/v1/attachment_upload_test.go new file mode 100644 index 00000000..40e81932 --- /dev/null +++ b/server/router/api/v1/attachment_upload_test.go @@ -0,0 +1,357 @@ +package v1 + +import ( + "bytes" + "context" + "io" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + + "github.com/usememos/memos/internal/testutil" + v1pb "github.com/usememos/memos/proto/gen/api/v1" + storepb "github.com/usememos/memos/proto/gen/store" + "github.com/usememos/memos/store" +) + +func newUploadTestService(t *testing.T) (*APIV1Service, context.Context) { + t.Helper() + svc := newIntegrationService(t) + t.Cleanup(svc.CloseAttachmentUploads) + user := createSpaceTestUser(context.Background(), t, svc, "uploader", store.RoleUser) + return svc, userCtx(context.Background(), user.ID) +} + +func uploadSpec(filename string, size int64) *v1pb.UploadAttachmentRequest_Spec { + return &v1pb.UploadAttachmentRequest_Spec{Spec: &v1pb.UploadAttachmentSpec{Attachment: &v1pb.Attachment{Filename: filename}, TotalSize: size}} +} + +func uploadID(id string) *v1pb.UploadAttachmentRequest_UploadId { + return &v1pb.UploadAttachmentRequest_UploadId{UploadId: id} +} + +func startTestUpload(ctx context.Context, t *testing.T, svc *APIV1Service, size int64) string { + t.Helper() + spec := uploadSpec("file.bin", size) + spec.Spec.Attachment.Type = "application/octet-stream" + response, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: spec}) + require.NoError(t, err) + require.NotEmpty(t, response.UploadId) + require.EqualValues(t, attachmentUploadChunkSize, response.MaxChunkSize) + require.Zero(t, response.CommittedSize) + require.Nil(t, response.Attachment) + return response.UploadId +} + +func TestUploadAttachmentChunksAndRetries(t *testing.T) { + svc, ctx := newUploadTestService(t) + id := startTestUpload(ctx, t, svc, 6) + first := &v1pb.UploadAttachmentRequest{Upload: uploadID(id), Data: []byte("abc")} + for range 2 { + response, err := svc.UploadAttachment(ctx, first) + require.NoError(t, err) + require.Equal(t, id, response.UploadId) + require.EqualValues(t, 3, response.CommittedSize) + } + for _, request := range []*v1pb.UploadAttachmentRequest{ + {Upload: uploadID(id), Data: []byte("xyz")}, + {Upload: uploadID(id), WriteOffset: 4, Data: []byte("d")}, + {Upload: uploadID(id), WriteOffset: -1, Data: []byte("d")}, + } { + _, err := svc.UploadAttachment(ctx, request) + require.Equal(t, codes.OutOfRange, status.Code(err)) + } + progress, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(id), WriteOffset: -1}) + require.NoError(t, err) + require.EqualValues(t, 3, progress.CommittedSize) + _, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(id), WriteOffset: 3, FinishWrite: true}) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + + last := &v1pb.UploadAttachmentRequest{Upload: uploadID(id), WriteOffset: 3, Data: []byte("def"), FinishWrite: true} + result, err := svc.UploadAttachment(ctx, last) + require.NoError(t, err) + require.EqualValues(t, 6, result.Attachment.Size) + for _, request := range []*v1pb.UploadAttachmentRequest{last, {Upload: uploadID(id)}, {Upload: uploadID(id), WriteOffset: 6, FinishWrite: true}} { + repeated, err := svc.UploadAttachment(ctx, request) + require.NoError(t, err) + require.Equal(t, result.Attachment.Name, repeated.Attachment.Name) + } + rows, err := svc.Store.ListAttachments(ctx, &store.FindAttachment{}) + require.NoError(t, err) + require.Len(t, rows, 1) + content, err := os.ReadFile(rows[0].Reference) + require.NoError(t, err) + require.Equal(t, "abcdef", string(content)) + require.NoFileExists(t, svc.attachmentUploads.entries[id].path) + + _, err = svc.DeleteAttachment(ctx, &v1pb.DeleteAttachmentRequest{Name: result.Attachment.Name}) + require.NoError(t, err) + _, err = svc.UploadAttachment(ctx, last) + require.Equal(t, codes.NotFound, status.Code(err)) +} + +func TestUploadAttachmentSingleCall(t *testing.T) { + svc, ctx := newUploadTestService(t) + spec := uploadSpec("note.txt", 3) + spec.Spec.AttachmentId = "chosen-id" + result, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: spec, Data: []byte("abc"), FinishWrite: true}) + require.NoError(t, err) + require.Equal(t, "attachments/chosen-id", result.Attachment.Name) + require.EqualValues(t, 3, result.Attachment.Size) + require.EqualValues(t, 3, result.CommittedSize) + + empty, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadSpec("empty.txt", 0), FinishWrite: true}) + require.NoError(t, err) + require.Zero(t, empty.Attachment.Size) + + // A spec call may carry the first chunk without finishing. + partial, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadSpec("two.bin", 4), Data: []byte("ab")}) + require.NoError(t, err) + require.Nil(t, partial.Attachment) + require.EqualValues(t, 2, partial.CommittedSize) + done, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(partial.UploadId), WriteOffset: 2, Data: []byte("cd"), FinishWrite: true}) + require.NoError(t, err) + require.EqualValues(t, 4, done.Attachment.Size) + + rows, err := svc.Store.ListAttachments(ctx, &store.FindAttachment{}) + require.NoError(t, err) + require.Len(t, rows, 3) +} + +func TestUploadAttachmentValidation(t *testing.T) { + svc, ctx := newUploadTestService(t) + _, err := svc.UploadAttachment(context.Background(), &v1pb.UploadAttachmentRequest{}) + require.Equal(t, codes.Unauthenticated, status.Code(err)) + badID := uploadSpec("file", 0) + badID.Spec.AttachmentId = "bad id!" + withContent := uploadSpec("file", 0) + withContent.Spec.Attachment.Content = []byte("not metadata") + for _, request := range []*v1pb.UploadAttachmentRequest{ + {}, + {Upload: &v1pb.UploadAttachmentRequest_Spec{Spec: &v1pb.UploadAttachmentSpec{}}}, + {Upload: uploadSpec("file", -1)}, + {Upload: uploadSpec("../file", 0)}, + {Upload: withContent}, + {Upload: badID}, + {Upload: uploadSpec(strings.Repeat("x", attachmentUploadMetadataLimit+1), 0)}, + {Upload: uploadSpec("file", 3), Data: []byte("four")}, + } { + _, err := svc.UploadAttachment(ctx, request) + require.Equal(t, codes.InvalidArgument, status.Code(err), request.String()) + } + _, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadSpec("file", 3), WriteOffset: 1, Data: []byte("abc")}) + require.Equal(t, codes.OutOfRange, status.Code(err)) + _, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadSpec("file", 1<<60)}) + require.Equal(t, codes.ResourceExhausted, status.Code(err)) + require.Empty(t, svc.attachmentUploads.entries, "rejected spec calls must not register uploads") + + id := startTestUpload(ctx, t, svc, 3) + other := createSpaceTestUser(context.Background(), t, svc, "other-uploader", store.RoleUser) + _, err = svc.UploadAttachment(userCtx(context.Background(), other.ID), &v1pb.UploadAttachmentRequest{Upload: uploadID(id)}) + require.Equal(t, codes.NotFound, status.Code(err)) + _, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID("unknown")}) + require.Equal(t, codes.NotFound, status.Code(err)) + _, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(id), Data: make([]byte, attachmentUploadChunkSize+1)}) + require.Equal(t, codes.ResourceExhausted, status.Code(err)) + _, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(id), Data: []byte("four")}) + require.Equal(t, codes.InvalidArgument, status.Code(err)) + canceled, cancel := context.WithCancel(ctx) + cancel() + _, err = svc.UploadAttachment(canceled, &v1pb.UploadAttachmentRequest{Upload: uploadID(id), Data: []byte("abc")}) + require.Error(t, err) + progress, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(id)}) + require.NoError(t, err) + require.Zero(t, progress.CommittedSize) +} + +func TestUploadAttachmentEmptyAndConcurrentFinish(t *testing.T) { + svc, ctx := newUploadTestService(t) + id := startTestUpload(ctx, t, svc, 0) + var wg sync.WaitGroup + results := make(chan *v1pb.UploadAttachmentResponse, 8) + errs := make(chan error, 8) + for range 8 { + wg.Go(func() { + response, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(id), FinishWrite: true}) + results <- response + errs <- err + }) + } + wg.Wait() + close(results) + close(errs) + for err := range errs { + require.NoError(t, err) + } + name := "" + for response := range results { + if name == "" { + name = response.Attachment.Name + } + require.Equal(t, name, response.Attachment.Name) + require.Zero(t, response.Attachment.Size) + } + rows, err := svc.Store.ListAttachments(ctx, &store.FindAttachment{}) + require.NoError(t, err) + require.Len(t, rows, 1) +} + +func TestUploadAttachmentFinalizationRecovery(t *testing.T) { + svc, ctx := newUploadTestService(t) + id := startTestUpload(ctx, t, svc, 3) + last := &v1pb.UploadAttachmentRequest{Upload: uploadID(id), Data: []byte("abc"), FinishWrite: true} + _, err := svc.UploadAttachment(store.WithCreateAttachmentPostCommitFailpoint(ctx), last) + require.Equal(t, codes.Internal, status.Code(err)) + result, err := svc.UploadAttachment(ctx, last) + require.NoError(t, err) + require.NotNil(t, result.Attachment) + rows, err := svc.Store.ListAttachments(ctx, &store.FindAttachment{}) + require.NoError(t, err) + require.Len(t, rows, 1) + content, err := os.ReadFile(rows[0].Reference) + require.NoError(t, err) + require.Equal(t, "abc", string(content)) +} + +func TestUploadAttachmentRechecksMemoAndSize(t *testing.T) { + svc, ctx := newUploadTestService(t) + memo, err := svc.CreateMemo(ctx, &v1pb.CreateMemoRequest{Memo: &v1pb.Memo{Content: "upload target"}}) + require.NoError(t, err) + spec := uploadSpec("file.txt", 3) + spec.Spec.Attachment.Memo = &memo.Name + initial, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: spec}) + require.NoError(t, err) + _, err = svc.DeleteMemo(ctx, &v1pb.DeleteMemoRequest{Name: memo.Name, Force: true}) + require.NoError(t, err) + _, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(initial.UploadId), Data: []byte("abc"), FinishWrite: true}) + require.Equal(t, codes.NotFound, status.Code(err)) + + id := startTestUpload(ctx, t, svc, 2*MebiByte) + _, err = svc.Store.UpsertInstanceSetting(ctx, &storepb.InstanceSetting{Key: storepb.InstanceSettingKey_STORAGE, + Value: &storepb.InstanceSetting_StorageSetting{StorageSetting: &storepb.InstanceStorageSetting{UploadSizeLimitMb: 1}}, + }) + require.NoError(t, err) + _, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(id), Data: make([]byte, 2*MebiByte), FinishWrite: true}) + require.Equal(t, codes.ResourceExhausted, status.Code(err)) + rows, err := svc.Store.ListAttachments(ctx, &store.FindAttachment{}) + require.NoError(t, err) + require.Empty(t, rows) +} + +func TestUploadAttachmentExpiryAndLimits(t *testing.T) { + svc, ctx := newUploadTestService(t) + ids := make([]string, attachmentUploadMaxActivePerUser) + for i := range ids { + ids[i] = startTestUpload(ctx, t, svc, 0) + } + _, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadSpec("too-many", 0)}) + require.Equal(t, codes.ResourceExhausted, status.Code(err)) + _, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(ids[0]), FinishWrite: true}) + require.NoError(t, err) + startTestUpload(ctx, t, svc, 0) // Completed uploads do not consume an active slot. + upload := svc.attachmentUploads.entries[ids[1]] + upload.mu.Lock() + upload.expireTime = time.Now().Add(-time.Second) + upload.mu.Unlock() + _, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(ids[1])}) + require.Equal(t, codes.NotFound, status.Code(err)) + + orphan := filepath.Join(svc.Profile.Data, attachmentUploadTempPrefix+"orphan") + recent := filepath.Join(svc.Profile.Data, attachmentUploadTempPrefix+"recent") + unrelated := filepath.Join(svc.Profile.Data, "keep.txt") + for _, path := range []string{orphan, recent, unrelated} { + require.NoError(t, os.WriteFile(path, []byte("x"), 0600)) + } + old := time.Now().Add(-2 * attachmentUploadTTL) + require.NoError(t, os.Chtimes(orphan, old, old)) + require.NoError(t, os.Chtimes(unrelated, old, old)) + svc.attachmentUploads.mu.Lock() + svc.attachmentUploads.sweepLocked(time.Now(), 0) + svc.attachmentUploads.removeOrphansLocked(svc.Profile.Data, time.Now()) + svc.attachmentUploads.mu.Unlock() + require.NoFileExists(t, upload.path) + require.NoFileExists(t, orphan) + require.FileExists(t, recent) + require.FileExists(t, unrelated) + svc.CloseAttachmentUploads() + require.Empty(t, svc.attachmentUploads.entries) +} + +func TestUploadAttachmentAboveLegacyRequestLimit(t *testing.T) { + svc, ctx := newUploadTestService(t) + const size = 300 * MebiByte + _, err := svc.Store.UpsertInstanceSetting(ctx, &storepb.InstanceSetting{Key: storepb.InstanceSettingKey_STORAGE, + Value: &storepb.InstanceSetting_StorageSetting{StorageSetting: &storepb.InstanceStorageSetting{UploadSizeLimitMb: 400}}, + }) + require.NoError(t, err) + id := startTestUpload(ctx, t, svc, size) + chunk := bytes.Repeat([]byte{0x5a}, attachmentUploadChunkSize) + var response *v1pb.UploadAttachmentResponse + for offset := int64(0); offset < size; offset += int64(len(chunk)) { + response, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{ + Upload: uploadID(id), WriteOffset: offset, Data: chunk, FinishWrite: offset+int64(len(chunk)) == size, + }) + require.NoError(t, err) + } + require.EqualValues(t, size, response.Attachment.Size) + rows, err := svc.Store.ListAttachments(ctx, &store.FindAttachment{}) + require.NoError(t, err) + require.Len(t, rows, 1) + file, err := os.Open(rows[0].Reference) + require.NoError(t, err) + defer file.Close() + info, err := file.Stat() + require.NoError(t, err) + require.EqualValues(t, size, info.Size()) + _, err = file.Seek(-int64(len(chunk)), io.SeekEnd) + require.NoError(t, err) + tail := make([]byte, len(chunk)) + _, err = io.ReadFull(file, tail) + require.NoError(t, err) + require.Equal(t, chunk, tail) +} + +func TestUploadAttachmentMediaProcessing(t *testing.T) { + for _, tc := range []struct { + name string + content []byte + motion bool + }{ + {name: "JPEG", content: testutil.BuildJPEG(20, 10)}, + {name: "motion photo", content: testutil.BuildMotionPhotoJPEG(), motion: true}, + } { + t.Run(tc.name, func(t *testing.T) { + svc, ctx := newUploadTestService(t) + metadata := &v1pb.Attachment{Filename: "image.unknown", MediaMetadata: &v1pb.MediaMetadata{Width: proto.Int32(20), Height: proto.Int32(10)}} + spec := &v1pb.UploadAttachmentRequest_Spec{Spec: &v1pb.UploadAttachmentSpec{Attachment: metadata, TotalSize: int64(len(tc.content))}} + initial, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: spec}) + require.NoError(t, err) + result, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(initial.UploadId), Data: tc.content, FinishWrite: true}) + require.NoError(t, err) + require.Equal(t, "image/jpeg", result.Attachment.Type) + require.True(t, proto.Equal(metadata.MediaMetadata, result.Attachment.MediaMetadata)) + uid, err := ExtractAttachmentUIDFromName(result.Attachment.Name) + require.NoError(t, err) + row, err := svc.Store.GetAttachment(ctx, &store.FindAttachment{UID: &uid}) + require.NoError(t, err) + content, err := os.ReadFile(row.Reference) + require.NoError(t, err) + if tc.motion { + require.Equal(t, tc.content, content) + require.True(t, result.Attachment.MotionMedia.HasEmbeddedVideo) + } else { + stripped, err := stripImageExif(bytes.NewReader(tc.content), "image/jpeg") + require.NoError(t, err) + require.Equal(t, stripped, content) + } + }) + } +} diff --git a/server/router/api/v1/attachment_upload_transport_test.go b/server/router/api/v1/attachment_upload_transport_test.go new file mode 100644 index 00000000..5dfee517 --- /dev/null +++ b/server/router/api/v1/attachment_upload_transport_test.go @@ -0,0 +1,111 @@ +package v1 + +import ( + "bytes" + "compress/gzip" + "context" + "io" + "net/http" + "net/http/httptest" + "testing" + + "connectrpc.com/connect" + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" + + v1pb "github.com/usememos/memos/proto/gen/api/v1" + "github.com/usememos/memos/proto/gen/api/v1/apiv1connect" + "github.com/usememos/memos/server/auth" +) + +type uploadCountingReader struct { + reader io.Reader + read int +} + +func (r *uploadCountingReader) Read(p []byte) (int, error) { + n, err := r.reader.Read(p) + r.read += n + return n, err +} + +func TestUploadAttachmentTransports(t *testing.T) { + svc, ctx := newUploadTestService(t) + user, err := svc.fetchCurrentUser(ctx) + require.NoError(t, err) + token, _, err := auth.GenerateAccessTokenV2(user.ID, user.Username, string(user.Role), string(user.RowStatus), []byte(svc.Secret)) + require.NoError(t, err) + e := echo.New() + require.NoError(t, svc.RegisterGateway(context.Background(), e)) + + for _, path := range []string{"/api/v1/attachments:upload", attachmentUploadProcedure} { + t.Run(path, func(t *testing.T) { + call := func(message proto.Message, authenticated bool) *httptest.ResponseRecorder { + t.Helper() + data, err := protojson.Marshal(message) + require.NoError(t, err) + req := httptest.NewRequest(http.MethodPost, path, bytes.NewReader(data)) + req.Header.Set("Content-Type", "application/json") + if authenticated { + req.Header.Set("Authorization", "Bearer "+token) + } + rec := httptest.NewRecorder() + e.ServeHTTP(rec, req) + return rec + } + initial := &v1pb.UploadAttachmentRequest{Upload: uploadSpec("file.txt", 3)} + require.Equal(t, http.StatusUnauthorized, call(initial, false).Code) + rec := call(initial, true) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + response := &v1pb.UploadAttachmentResponse{} + require.NoError(t, protojson.Unmarshal(rec.Body.Bytes(), response)) + rec = call(&v1pb.UploadAttachmentRequest{Upload: uploadID(response.UploadId), Data: []byte("abc"), FinishWrite: true}, true) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + require.NoError(t, protojson.Unmarshal(rec.Body.Bytes(), response)) + require.NotNil(t, response.Attachment) + + oversized, err := protojson.Marshal(&v1pb.UploadAttachmentRequest{Data: make([]byte, attachmentUploadRequestLimit)}) + require.NoError(t, err) + reader := &uploadCountingReader{reader: bytes.NewReader(oversized)} + req := httptest.NewRequest(http.MethodPost, path, reader) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+token) + rec = httptest.NewRecorder() + e.ServeHTTP(rec, req) + require.GreaterOrEqual(t, rec.Code, 400) + require.LessOrEqual(t, reader.read, attachmentUploadRequestLimit+1, "body must be bounded before decoding") + }) + } + + t.Run("Connect protobuf and compressed message limit", func(t *testing.T) { + server := httptest.NewTestServer(t, e) + client := apiv1connect.NewAttachmentServiceClient(server.Client(), server.URL) + initial := connect.NewRequest(&v1pb.UploadAttachmentRequest{Upload: uploadSpec("binary.bin", attachmentUploadChunkSize)}) + initial.Header().Set("Authorization", "Bearer "+token) + response, err := client.UploadAttachment(context.Background(), initial) + require.NoError(t, err) + last := connect.NewRequest(&v1pb.UploadAttachmentRequest{Upload: uploadID(response.Msg.UploadId), Data: make([]byte, attachmentUploadChunkSize), FinishWrite: true}) + last.Header().Set("Authorization", "Bearer "+token) + finished, err := client.UploadAttachment(context.Background(), last) + require.NoError(t, err) + require.EqualValues(t, attachmentUploadChunkSize, finished.Msg.Attachment.Size) + + data, err := proto.Marshal(&v1pb.UploadAttachmentRequest{Data: make([]byte, attachmentUploadRequestLimit+1)}) + require.NoError(t, err) + var compressed bytes.Buffer + writer := gzip.NewWriter(&compressed) + _, err = writer.Write(data) + require.NoError(t, err) + require.NoError(t, writer.Close()) + req := httptest.NewRequest(http.MethodPost, attachmentUploadProcedure, &compressed) + req.Header.Set("Content-Type", "application/proto") + req.Header.Set("Content-Encoding", "gzip") + req.Header.Set("Authorization", "Bearer "+token) + rec := httptest.NewRecorder() + e.ServeHTTP(rec, req) + require.Equal(t, http.StatusTooManyRequests, rec.Code, rec.Body.String()) + require.Contains(t, rec.Body.String(), "resource_exhausted") + }) +} diff --git a/server/router/api/v1/connect_services.go b/server/router/api/v1/connect_services.go index 291326f2..37d19377 100644 --- a/server/router/api/v1/connect_services.go +++ b/server/router/api/v1/connect_services.go @@ -595,6 +595,14 @@ func (s *ConnectServiceHandler) DeleteSpaceMember(ctx context.Context, req *conn // AttachmentService +func (s *ConnectServiceHandler) UploadAttachment(ctx context.Context, req *connect.Request[v1pb.UploadAttachmentRequest]) (*connect.Response[v1pb.UploadAttachmentResponse], error) { + resp, err := s.APIV1Service.UploadAttachment(ctx, req.Msg) + if err != nil { + return nil, convertGRPCError(err) + } + return connect.NewResponse(resp), nil +} + func (s *ConnectServiceHandler) CreateAttachment(ctx context.Context, req *connect.Request[v1pb.CreateAttachmentRequest]) (*connect.Response[v1pb.Attachment], error) { resp, err := s.APIV1Service.CreateAttachment(ctx, req.Msg) if err != nil { diff --git a/server/router/api/v1/test/attachment_service_s3_test.go b/server/router/api/v1/test/attachment_service_s3_test.go index a684a56e..a606243a 100644 --- a/server/router/api/v1/test/attachment_service_s3_test.go +++ b/server/router/api/v1/test/attachment_service_s3_test.go @@ -21,6 +21,40 @@ func TestS3AttachmentLifecycleAcrossStorageChange(t *testing.T) { runS3AttachmentLifecycleAcrossStorageChange(t, fake) } +func TestUploadAttachmentS3(t *testing.T) { + ts := NewTestService(t) + defer ts.Cleanup() + defer ts.Service.CloseAttachmentUploads() + fake := fakes3.New(t, "uploads") + ctx := context.Background() + user, err := ts.CreateRegularUser(ctx, "chunk-uploader") + require.NoError(t, err) + ctx = ts.CreateUserContext(ctx, user.ID) + storage := fakeStorage("upload-s3", "Uploads", fake.Config("uploads")) + upsertS3StorageSetting(ctx, t, ts, storage.Id, storage) + chunk := bytes.Repeat([]byte("x"), 2<<20) + initial, err := ts.Service.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: &v1pb.UploadAttachmentRequest_Spec{Spec: &v1pb.UploadAttachmentSpec{ + Attachment: &v1pb.Attachment{Filename: "large.bin", Type: "application/octet-stream"}, TotalSize: int64(3 * len(chunk)), + }}}) + require.NoError(t, err) + var result *v1pb.UploadAttachmentResponse + for i := range 3 { + result, err = ts.Service.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{ + Upload: &v1pb.UploadAttachmentRequest_UploadId{UploadId: initial.UploadId}, WriteOffset: int64(i * len(chunk)), Data: chunk, FinishWrite: i == 2, + }) + require.NoError(t, err) + } + uid, err := apiv1.ExtractAttachmentUIDFromName(result.Attachment.Name) + require.NoError(t, err) + row, err := ts.Store.GetAttachment(ctx, &store.FindAttachment{UID: &uid}) + require.NoError(t, err) + require.Empty(t, row.Blob) + require.Equal(t, storepb.AttachmentStorageType_S3, row.StorageType) + content, err := fake.GetObject("uploads", row.Payload.GetS3Object().GetKey()) + require.NoError(t, err) + require.Equal(t, bytes.Repeat(chunk, 3), content) +} + func TestS3AttachmentLifecycleAcrossStorageChangeMinIO(t *testing.T) { server := testminio.New(t, "attachments-old", "attachments-new") runS3AttachmentLifecycleAcrossStorageChange(t, server) diff --git a/server/router/api/v1/test/memo_timestamp_test.go b/server/router/api/v1/test/memo_timestamp_test.go index fc057db7..74e5e835 100644 --- a/server/router/api/v1/test/memo_timestamp_test.go +++ b/server/router/api/v1/test/memo_timestamp_test.go @@ -49,14 +49,14 @@ func TestUpdateMemoTimestamps(t *testing.T) { }}) require.NoError(t, err) tc.patch.Name = memo.Name - before := time.Now().Unix() + beforeSec := time.Now().Unix() updated, err := ts.Service.UpdateMemo(userCtx, &apiv1.UpdateMemoRequest{ Memo: tc.patch, UpdateMask: &fieldmaskpb.FieldMask{Paths: tc.paths}, }) require.NoError(t, err) require.Equal(t, tc.wantCreate, updated.CreateTime.Seconds) if tc.autoUpdate { - require.GreaterOrEqual(t, updated.UpdateTime.Seconds, before) + require.GreaterOrEqual(t, updated.UpdateTime.Seconds, beforeSec) require.LessOrEqual(t, updated.UpdateTime.Seconds, time.Now().Unix()) } else { require.Equal(t, tc.wantUpdate, updated.UpdateTime.Seconds) diff --git a/server/router/api/v1/v1.go b/server/router/api/v1/v1.go index 47178ea9..135bf089 100644 --- a/server/router/api/v1/v1.go +++ b/server/router/api/v1/v1.go @@ -25,6 +25,15 @@ import ( // it derives its own limit from this constant to keep the two gates in lockstep. const MaxAPIRequestBytes = 256 << 20 +// requestBodyLimit returns the request body cap for a procedure. Chunked +// uploads carry at most one chunk per call, so they get a much lower cap. +func requestBodyLimit(procedure string) int64 { + if procedure == attachmentUploadProcedure { + return attachmentUploadRequestLimit + } + return MaxAPIRequestBytes +} + type APIV1Service struct { v1pb.UnimplementedInstanceServiceServer v1pb.UnimplementedAuthServiceServer @@ -51,6 +60,7 @@ type APIV1Service struct { instanceStatsCache instanceStatsCache linkMetadataFetcher linkMetadataFetcher + attachmentUploads attachmentUploads } // NewAPIV1Service creates an API v1 service with its shared dependencies. @@ -132,6 +142,7 @@ func (s *APIV1Service) RegisterGateway(ctx context.Context, echoServer *echo.Ech if result != nil { r = r.WithContext(auth.ApplyToContext(ctx, result)) } + r.Body = http.MaxBytesReader(w, r.Body, requestBodyLimit(procedure)) next(w, r, pathParams) } @@ -172,7 +183,7 @@ func (s *APIV1Service) RegisterGateway(ctx context.Context, echoServer *echo.Ech gwGroup := echoServer.Group("") // Register SSE endpoint with same CORS as rest of /api/v1. RegisterSSERoutes(gwGroup, s.SSEHub, s.Store, s.Secret) - handler := echo.WrapHandler(http.MaxBytesHandler(gwMux, MaxAPIRequestBytes)) + handler := echo.WrapHandler(gwMux) gwGroup.Any("/api/v1/*", handler) gwGroup.Any("/file/*", handler) @@ -187,10 +198,17 @@ func (s *APIV1Service) RegisterGateway(ctx context.Context, echoServer *echo.Ech ) connectMux := http.NewServeMux() connectHandler := NewConnectServiceHandler(s) - connectHandler.RegisterConnectHandlers(connectMux, connectInterceptors, connect.WithReadMaxBytes(MaxAPIRequestBytes)) + connectHandler.RegisterConnectHandlers(connectMux, connectInterceptors, + // Bound the decompressed message as well as the wire bytes below. + connect.WithConditionalHandlerOptions(func(spec connect.Spec) []connect.HandlerOption { + return []connect.HandlerOption{connect.WithReadMaxBytes(int(requestBodyLimit(spec.Procedure)))} + })) connectGroup := echoServer.Group("") - connectGroup.Any("/memos.api.v1.*", echo.WrapHandler(http.MaxBytesHandler(connectMux, MaxAPIRequestBytes))) + connectGroup.Any("/memos.api.v1.*", echo.WrapHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, requestBodyLimit(r.URL.Path)) + connectMux.ServeHTTP(w, r) + }))) return nil } diff --git a/server/server.go b/server/server.go index 7c81279d..9a528cce 100644 --- a/server/server.go +++ b/server/server.go @@ -30,9 +30,9 @@ type Server struct { Profile *profile.Profile Store *store.Store - echoServer *echo.Echo - httpServer *http.Server - sseHub *apiv1.SSEHub + echoServer *echo.Echo + httpServer *http.Server + apiV1Service *apiv1.APIV1Service } func NewServer(ctx context.Context, profile *profile.Profile, store *store.Store) (*Server, error) { @@ -66,7 +66,7 @@ func NewServer(ctx context.Context, profile *profile.Profile, store *store.Store frontend.NewFrontendService(profile, store).Serve(ctx, echoServer) apiV1Service := apiv1.NewAPIV1Service(s.Secret, profile, store) - s.sseHub = apiV1Service.SSEHub + s.apiV1Service = apiV1Service // Register HTTP file server routes BEFORE gRPC-Gateway to ensure proper range request handling for Safari. // This uses native HTTP serving (http.ServeContent) instead of gRPC for video/audio files. @@ -127,6 +127,7 @@ func (s *Server) Shutdown(ctx context.Context) { s.closeLongLivedConnections() s.shutdownHTTPServer(ctx) + s.apiV1Service.CloseAttachmentUploads() // Close database connection. if err := s.Store.Close(); err != nil { @@ -138,9 +139,7 @@ func (s *Server) Shutdown(ctx context.Context) { func (s *Server) closeLongLivedConnections() { // Long-lived SSE requests do not finish on their own during http.Server.Shutdown. - if s.sseHub != nil { - s.sseHub.Close() - } + s.apiV1Service.SSEHub.Close() } func (s *Server) shutdownHTTPServer(ctx context.Context) { diff --git a/web/src/components/MemoEditor/services/uploadService.ts b/web/src/components/MemoEditor/services/uploadService.ts index 9063b56f..daf6cd28 100644 --- a/web/src/components/MemoEditor/services/uploadService.ts +++ b/web/src/components/MemoEditor/services/uploadService.ts @@ -1,35 +1,74 @@ import { create } from "@bufbuild/protobuf"; +import { Code, ConnectError } from "@connectrpc/connect"; import { attachmentServiceClient } from "@/connect"; -import type { Attachment } from "@/types/proto/api/v1/attachment_service_pb"; -import { AttachmentSchema, MotionMediaSchema } from "@/types/proto/api/v1/attachment_service_pb"; +import type { Attachment, UploadAttachmentRequest } from "@/types/proto/api/v1/attachment_service_pb"; +import { AttachmentSchema, MotionMediaSchema, UploadAttachmentSpecSchema } from "@/types/proto/api/v1/attachment_service_pb"; import type { LocalFile } from "../types/attachment"; +const DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024; +// The server allows eight unfinished uploads per user; stay well under it. +const MAX_CONCURRENT_UPLOADS = 4; + export const uploadService = { - async uploadFile(localFile: LocalFile): Promise { + async uploadFile(localFile: LocalFile, signal?: AbortSignal): Promise { const { file, motionMedia } = localFile; - const [mediaMetadata, arrayBuffer] = await Promise.all([localFile.mediaMetadata, file.arrayBuffer()]); - const buffer = new Uint8Array(arrayBuffer); - return attachmentServiceClient.createAttachment({ + const mediaMetadata = await localFile.mediaMetadata; + const spec = create(UploadAttachmentSpecSchema, { attachment: create(AttachmentSchema, { filename: file.name, - size: BigInt(file.size), type: file.type, - content: buffer, motionMedia: motionMedia ? create(MotionMediaSchema, motionMedia) : undefined, mediaMetadata, }), + totalSize: BigInt(file.size), }); + // The spec call carries no data so that retrying it after a lost response + // can at worst orphan an upload, never create a duplicate attachment. + const initial = await attachmentServiceClient.uploadAttachment({ upload: { case: "spec", value: spec } }, { signal }); + if (!initial.uploadId || initial.committedSize !== 0n || initial.maxChunkSize <= 0) { + throw new Error("Invalid upload initialization response"); + } + const chunkSize = Math.min(initial.maxChunkSize, DEFAULT_CHUNK_SIZE); + for (let offset = 0; ; ) { + signal?.throwIfAborted(); + const end = Math.min(offset + chunkSize, file.size); + const request: Pick = { + upload: { case: "uploadId", value: initial.uploadId }, + writeOffset: BigInt(offset), + data: new Uint8Array(await file.slice(offset, end).arrayBuffer()), + finishWrite: end === file.size, + }; + // Retrying the identical last write is safe even if its response was lost + // after the server finalized the attachment. + let response; + for (let attempt = 0; ; attempt++) { + try { + response = await attachmentServiceClient.uploadAttachment(request, { signal }); + break; + } catch (error) { + if (signal?.aborted || ConnectError.from(error).code !== Code.Unavailable || attempt >= 2) throw error; + } + } + if (response.committedSize !== BigInt(end)) { + throw new Error("Unexpected upload offset"); + } + if (request.finishWrite) { + if (!response.attachment) throw new Error("Upload completed without an attachment"); + return response.attachment; + } + offset = end; + } }, async uploadFiles(localFiles: LocalFile[]): Promise { - if (localFiles.length === 0) return []; - - const attachments: Attachment[] = []; - - for (const localFile of localFiles) { - attachments.push(await uploadService.uploadFile(localFile)); - } - + const attachments: Attachment[] = new Array(localFiles.length); + let next = 0; + const worker = async () => { + for (let index = next++; index < localFiles.length; index = next++) { + attachments[index] = await this.uploadFile(localFiles[index]); + } + }; + await Promise.all(Array.from({ length: Math.min(MAX_CONCURRENT_UPLOADS, localFiles.length) }, worker)); return attachments; }, }; diff --git a/web/src/types/proto/api/v1/attachment_service_pb.ts b/web/src/types/proto/api/v1/attachment_service_pb.ts index 17e12bd9..a6a76722 100644 --- a/web/src/types/proto/api/v1/attachment_service_pb.ts +++ b/web/src/types/proto/api/v1/attachment_service_pb.ts @@ -16,7 +16,7 @@ import type { Message } from "@bufbuild/protobuf"; * Describes the file api/v1/attachment_service.proto. */ export const file_api_v1_attachment_service: GenFile = /*@__PURE__*/ - fileDesc("Ch9hcGkvdjEvYXR0YWNobWVudF9zZXJ2aWNlLnByb3RvEgxtZW1vcy5hcGkudjEivAEKC01vdGlvbk1lZGlhEi8KBmZhbWlseRgBIAEoDjIfLm1lbW9zLmFwaS52MS5Nb3Rpb25NZWRpYUZhbWlseRIrCgRyb2xlGAIgASgOMh0ubWVtb3MuYXBpLnYxLk1vdGlvbk1lZGlhUm9sZRIQCghncm91cF9pZBgDIAEoCRIhChlwcmVzZW50YXRpb25fdGltZXN0YW1wX3VzGAQgASgDEhoKEmhhc19lbWJlZGRlZF92aWRlbxgFIAEoCCK0AQoNTWVkaWFNZXRhZGF0YRISCgV3aWR0aBgBIAEoBUgBiAEBEhMKBmhlaWdodBgCIAEoBUgCiAEBEiwKBXBob3RvGAMgASgLMhsubWVtb3MuYXBpLnYxLlBob3RvTWV0YWRhdGFIABIsCgV2aWRlbxgEIAEoCzIbLm1lbW9zLmFwaS52MS5WaWRlb01ldGFkYXRhSABCCQoHZGV0YWlsc0IICgZfd2lkdGhCCQoHX2hlaWdodCKjAwoNUGhvdG9NZXRhZGF0YRI0CgxjYXB0dXJlX3RpbWUYASABKAsyHi5tZW1vcy5hcGkudjEuTWVkaWFDYXB0dXJlVGltZRItCghsb2NhdGlvbhgCIAEoCzIbLm1lbW9zLmFwaS52MS5NZWRpYUxvY2F0aW9uEiQKF3NvdXJjZV9leGlmX29yaWVudGF0aW9uGAMgASgFSACIAQESEwoLY2FtZXJhX21ha2UYBCABKAkSFAoMY2FtZXJhX21vZGVsGAUgASgJEhIKCmxlbnNfbW9kZWwYBiABKAkSFQoIZl9udW1iZXIYByABKAFIAYgBARIiChVleHBvc3VyZV90aW1lX3NlY29uZHMYCCABKAFIAogBARIQCgNpc28YCSABKAVIA4gBARIcCg9mb2NhbF9sZW5ndGhfbW0YCiABKAFIBIgBAUIaChhfc291cmNlX2V4aWZfb3JpZW50YXRpb25CCwoJX2ZfbnVtYmVyQhgKFl9leHBvc3VyZV90aW1lX3NlY29uZHNCBgoEX2lzb0ISChBfZm9jYWxfbGVuZ3RoX21tIlMKEE1lZGlhQ2FwdHVyZVRpbWUSFwoPbG9jYWxfZGF0ZV90aW1lGAEgASgJEhcKCnV0Y19vZmZzZXQYAiABKAlIAIgBAUINCgtfdXRjX29mZnNldCKLAQoNTWVkaWFMb2NhdGlvbhIVCghsYXRpdHVkZRgBIAEoAUgAiAEBEhYKCWxvbmdpdHVkZRgCIAEoAUgBiAEBEhwKD2FsdGl0dWRlX21ldGVycxgDIAEoAUgCiAEBQgsKCV9sYXRpdHVkZUIMCgpfbG9uZ2l0dWRlQhIKEF9hbHRpdHVkZV9tZXRlcnMiQwoNVmlkZW9NZXRhZGF0YRIdChBkdXJhdGlvbl9zZWNvbmRzGAEgASgBSACIAQFCEwoRX2R1cmF0aW9uX3NlY29uZHMiqQMKCkF0dGFjaG1lbnQSEQoEbmFtZRgBIAEoCUID4EEIEjQKC2NyZWF0ZV90aW1lGAIgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcEID4EEDEhUKCGZpbGVuYW1lGAMgASgJQgPgQQISFAoHY29udGVudBgEIAEoDEID4EEEEhoKDWV4dGVybmFsX2xpbmsYBSABKAlCA+BBARIRCgR0eXBlGAYgASgJQgPgQQISEQoEc2l6ZRgHIAEoA0ID4EEDEhYKBG1lbW8YCCABKAlCA+BBAUgAiAEBEjQKDG1vdGlvbl9tZWRpYRgJIAEoCzIZLm1lbW9zLmFwaS52MS5Nb3Rpb25NZWRpYUID4EEBEjsKDm1lZGlhX21ldGFkYXRhGAogASgLMhsubWVtb3MuYXBpLnYxLk1lZGlhTWV0YWRhdGFCBuBBAeBBBTpP6kFMChdtZW1vcy5hcGkudjEvQXR0YWNobWVudBIYYXR0YWNobWVudHMve2F0dGFjaG1lbnR9KgthdHRhY2htZW50czIKYXR0YWNobWVudEIHCgVfbWVtbyJoChdDcmVhdGVBdHRhY2htZW50UmVxdWVzdBIxCgphdHRhY2htZW50GAEgASgLMhgubWVtb3MuYXBpLnYxLkF0dGFjaG1lbnRCA+BBAhIaCg1hdHRhY2htZW50X2lkGAIgASgJQgPgQQEilAEKFkxpc3RBdHRhY2htZW50c1JlcXVlc3QSFgoJcGFnZV9zaXplGAEgASgFQgPgQQESFwoKcGFnZV90b2tlbhgCIAEoCUID4EEBEhMKBmZpbHRlchgDIAEoCUID4EEBEhUKCG9yZGVyX2J5GAQgASgJQgPgQQFKBAgFEAZKBAgGEAdSBXNwYWNlUgp1bmFzc2lnbmVkImEKF0xpc3RBdHRhY2htZW50c1Jlc3BvbnNlEi0KC2F0dGFjaG1lbnRzGAEgAygLMhgubWVtb3MuYXBpLnYxLkF0dGFjaG1lbnQSFwoPbmV4dF9wYWdlX3Rva2VuGAIgASgJIkUKFEdldEF0dGFjaG1lbnRSZXF1ZXN0Ei0KBG5hbWUYASABKAlCH+BBAvpBGQoXbWVtb3MuYXBpLnYxL0F0dGFjaG1lbnQiggEKF1VwZGF0ZUF0dGFjaG1lbnRSZXF1ZXN0EjEKCmF0dGFjaG1lbnQYASABKAsyGC5tZW1vcy5hcGkudjEuQXR0YWNobWVudEID4EECEjQKC3VwZGF0ZV9tYXNrGAIgASgLMhouZ29vZ2xlLnByb3RvYnVmLkZpZWxkTWFza0ID4EECIkgKF0RlbGV0ZUF0dGFjaG1lbnRSZXF1ZXN0Ei0KBG5hbWUYASABKAlCH+BBAvpBGQoXbWVtb3MuYXBpLnYxL0F0dGFjaG1lbnQiMwodQmF0Y2hEZWxldGVBdHRhY2htZW50c1JlcXVlc3QSEgoFbmFtZXMYASADKAlCA+BBAipoChFNb3Rpb25NZWRpYUZhbWlseRIjCh9NT1RJT05fTUVESUFfRkFNSUxZX1VOU1BFQ0lGSUVEEAASFAoQQVBQTEVfTElWRV9QSE9UTxABEhgKFEFORFJPSURfTU9USU9OX1BIT1RPEAIqWQoPTW90aW9uTWVkaWFSb2xlEiEKHU1PVElPTl9NRURJQV9ST0xFX1VOU1BFQ0lGSUVEEAASCQoFU1RJTEwQARIJCgVWSURFTxACEg0KCUNPTlRBSU5FUhADMtAGChFBdHRhY2htZW50U2VydmljZRKJAQoQQ3JlYXRlQXR0YWNobWVudBIlLm1lbW9zLmFwaS52MS5DcmVhdGVBdHRhY2htZW50UmVxdWVzdBoYLm1lbW9zLmFwaS52MS5BdHRhY2htZW50IjTaQQphdHRhY2htZW50gtPkkwIhOgphdHRhY2htZW50IhMvYXBpL3YxL2F0dGFjaG1lbnRzEnsKD0xpc3RBdHRhY2htZW50cxIkLm1lbW9zLmFwaS52MS5MaXN0QXR0YWNobWVudHNSZXF1ZXN0GiUubWVtb3MuYXBpLnYxLkxpc3RBdHRhY2htZW50c1Jlc3BvbnNlIhuC0+STAhUSEy9hcGkvdjEvYXR0YWNobWVudHMSegoNR2V0QXR0YWNobWVudBIiLm1lbW9zLmFwaS52MS5HZXRBdHRhY2htZW50UmVxdWVzdBoYLm1lbW9zLmFwaS52MS5BdHRhY2htZW50IivaQQRuYW1lgtPkkwIeEhwvYXBpL3YxL3tuYW1lPWF0dGFjaG1lbnRzLyp9EqkBChBVcGRhdGVBdHRhY2htZW50EiUubWVtb3MuYXBpLnYxLlVwZGF0ZUF0dGFjaG1lbnRSZXF1ZXN0GhgubWVtb3MuYXBpLnYxLkF0dGFjaG1lbnQiVNpBFmF0dGFjaG1lbnQsdXBkYXRlX21hc2uC0+STAjU6CmF0dGFjaG1lbnQyJy9hcGkvdjEve2F0dGFjaG1lbnQubmFtZT1hdHRhY2htZW50cy8qfRJ+ChBEZWxldGVBdHRhY2htZW50EiUubWVtb3MuYXBpLnYxLkRlbGV0ZUF0dGFjaG1lbnRSZXF1ZXN0GhYuZ29vZ2xlLnByb3RvYnVmLkVtcHR5IivaQQRuYW1lgtPkkwIeKhwvYXBpL3YxL3tuYW1lPWF0dGFjaG1lbnRzLyp9EokBChZCYXRjaERlbGV0ZUF0dGFjaG1lbnRzEisubWVtb3MuYXBpLnYxLkJhdGNoRGVsZXRlQXR0YWNobWVudHNSZXF1ZXN0GhYuZ29vZ2xlLnByb3RvYnVmLkVtcHR5IiqC0+STAiQ6ASoiHy9hcGkvdjEvYXR0YWNobWVudHM6YmF0Y2hEZWxldGVCrgEKEGNvbS5tZW1vcy5hcGkudjFCFkF0dGFjaG1lbnRTZXJ2aWNlUHJvdG9QAVowZ2l0aHViLmNvbS91c2VtZW1vcy9tZW1vcy9wcm90by9nZW4vYXBpL3YxO2FwaXYxogIDTUFYqgIMTWVtb3MuQXBpLlYxygIMTWVtb3NcQXBpXFYx4gIYTWVtb3NcQXBpXFYxXEdQQk1ldGFkYXRh6gIOTWVtb3M6OkFwaTo6VjFiBnByb3RvMw", [file_google_api_annotations, file_google_api_client, file_google_api_field_behavior, file_google_api_resource, file_google_protobuf_empty, file_google_protobuf_field_mask, file_google_protobuf_timestamp]); + fileDesc("Ch9hcGkvdjEvYXR0YWNobWVudF9zZXJ2aWNlLnByb3RvEgxtZW1vcy5hcGkudjEivAEKC01vdGlvbk1lZGlhEi8KBmZhbWlseRgBIAEoDjIfLm1lbW9zLmFwaS52MS5Nb3Rpb25NZWRpYUZhbWlseRIrCgRyb2xlGAIgASgOMh0ubWVtb3MuYXBpLnYxLk1vdGlvbk1lZGlhUm9sZRIQCghncm91cF9pZBgDIAEoCRIhChlwcmVzZW50YXRpb25fdGltZXN0YW1wX3VzGAQgASgDEhoKEmhhc19lbWJlZGRlZF92aWRlbxgFIAEoCCK0AQoNTWVkaWFNZXRhZGF0YRISCgV3aWR0aBgBIAEoBUgBiAEBEhMKBmhlaWdodBgCIAEoBUgCiAEBEiwKBXBob3RvGAMgASgLMhsubWVtb3MuYXBpLnYxLlBob3RvTWV0YWRhdGFIABIsCgV2aWRlbxgEIAEoCzIbLm1lbW9zLmFwaS52MS5WaWRlb01ldGFkYXRhSABCCQoHZGV0YWlsc0IICgZfd2lkdGhCCQoHX2hlaWdodCKjAwoNUGhvdG9NZXRhZGF0YRI0CgxjYXB0dXJlX3RpbWUYASABKAsyHi5tZW1vcy5hcGkudjEuTWVkaWFDYXB0dXJlVGltZRItCghsb2NhdGlvbhgCIAEoCzIbLm1lbW9zLmFwaS52MS5NZWRpYUxvY2F0aW9uEiQKF3NvdXJjZV9leGlmX29yaWVudGF0aW9uGAMgASgFSACIAQESEwoLY2FtZXJhX21ha2UYBCABKAkSFAoMY2FtZXJhX21vZGVsGAUgASgJEhIKCmxlbnNfbW9kZWwYBiABKAkSFQoIZl9udW1iZXIYByABKAFIAYgBARIiChVleHBvc3VyZV90aW1lX3NlY29uZHMYCCABKAFIAogBARIQCgNpc28YCSABKAVIA4gBARIcCg9mb2NhbF9sZW5ndGhfbW0YCiABKAFIBIgBAUIaChhfc291cmNlX2V4aWZfb3JpZW50YXRpb25CCwoJX2ZfbnVtYmVyQhgKFl9leHBvc3VyZV90aW1lX3NlY29uZHNCBgoEX2lzb0ISChBfZm9jYWxfbGVuZ3RoX21tIlMKEE1lZGlhQ2FwdHVyZVRpbWUSFwoPbG9jYWxfZGF0ZV90aW1lGAEgASgJEhcKCnV0Y19vZmZzZXQYAiABKAlIAIgBAUINCgtfdXRjX29mZnNldCKLAQoNTWVkaWFMb2NhdGlvbhIVCghsYXRpdHVkZRgBIAEoAUgAiAEBEhYKCWxvbmdpdHVkZRgCIAEoAUgBiAEBEhwKD2FsdGl0dWRlX21ldGVycxgDIAEoAUgCiAEBQgsKCV9sYXRpdHVkZUIMCgpfbG9uZ2l0dWRlQhIKEF9hbHRpdHVkZV9tZXRlcnMiQwoNVmlkZW9NZXRhZGF0YRIdChBkdXJhdGlvbl9zZWNvbmRzGAEgASgBSACIAQFCEwoRX2R1cmF0aW9uX3NlY29uZHMiqQMKCkF0dGFjaG1lbnQSEQoEbmFtZRgBIAEoCUID4EEIEjQKC2NyZWF0ZV90aW1lGAIgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcEID4EEDEhUKCGZpbGVuYW1lGAMgASgJQgPgQQISFAoHY29udGVudBgEIAEoDEID4EEEEhoKDWV4dGVybmFsX2xpbmsYBSABKAlCA+BBARIRCgR0eXBlGAYgASgJQgPgQQISEQoEc2l6ZRgHIAEoA0ID4EEDEhYKBG1lbW8YCCABKAlCA+BBAUgAiAEBEjQKDG1vdGlvbl9tZWRpYRgJIAEoCzIZLm1lbW9zLmFwaS52MS5Nb3Rpb25NZWRpYUID4EEBEjsKDm1lZGlhX21ldGFkYXRhGAogASgLMhsubWVtb3MuYXBpLnYxLk1lZGlhTWV0YWRhdGFCBuBBAeBBBTpP6kFMChdtZW1vcy5hcGkudjEvQXR0YWNobWVudBIYYXR0YWNobWVudHMve2F0dGFjaG1lbnR9KgthdHRhY2htZW50czIKYXR0YWNobWVudEIHCgVfbWVtbyJoChdDcmVhdGVBdHRhY2htZW50UmVxdWVzdBIxCgphdHRhY2htZW50GAEgASgLMhgubWVtb3MuYXBpLnYxLkF0dGFjaG1lbnRCA+BBAhIaCg1hdHRhY2htZW50X2lkGAIgASgJQgPgQQEitQEKF1VwbG9hZEF0dGFjaG1lbnRSZXF1ZXN0EjIKBHNwZWMYASABKAsyIi5tZW1vcy5hcGkudjEuVXBsb2FkQXR0YWNobWVudFNwZWNIABITCgl1cGxvYWRfaWQYAiABKAlIABIZCgx3cml0ZV9vZmZzZXQYAyABKANCA+BBAhIRCgRkYXRhGAQgASgMQgPgQQESGQoMZmluaXNoX3dyaXRlGAUgASgIQgPgQQFCCAoGdXBsb2FkIn4KFFVwbG9hZEF0dGFjaG1lbnRTcGVjEjEKCmF0dGFjaG1lbnQYASABKAsyGC5tZW1vcy5hcGkudjEuQXR0YWNobWVudEID4EECEhoKDWF0dGFjaG1lbnRfaWQYAiABKAlCA+BBARIXCgp0b3RhbF9zaXplGAMgASgDQgPgQQEiiwEKGFVwbG9hZEF0dGFjaG1lbnRSZXNwb25zZRIRCgl1cGxvYWRfaWQYASABKAkSFgoOY29tbWl0dGVkX3NpemUYAiABKAMSLAoKYXR0YWNobWVudBgDIAEoCzIYLm1lbW9zLmFwaS52MS5BdHRhY2htZW50EhYKDm1heF9jaHVua19zaXplGAQgASgFIpQBChZMaXN0QXR0YWNobWVudHNSZXF1ZXN0EhYKCXBhZ2Vfc2l6ZRgBIAEoBUID4EEBEhcKCnBhZ2VfdG9rZW4YAiABKAlCA+BBARITCgZmaWx0ZXIYAyABKAlCA+BBARIVCghvcmRlcl9ieRgEIAEoCUID4EEBSgQIBRAGSgQIBhAHUgVzcGFjZVIKdW5hc3NpZ25lZCJhChdMaXN0QXR0YWNobWVudHNSZXNwb25zZRItCgthdHRhY2htZW50cxgBIAMoCzIYLm1lbW9zLmFwaS52MS5BdHRhY2htZW50EhcKD25leHRfcGFnZV90b2tlbhgCIAEoCSJFChRHZXRBdHRhY2htZW50UmVxdWVzdBItCgRuYW1lGAEgASgJQh/gQQL6QRkKF21lbW9zLmFwaS52MS9BdHRhY2htZW50IoIBChdVcGRhdGVBdHRhY2htZW50UmVxdWVzdBIxCgphdHRhY2htZW50GAEgASgLMhgubWVtb3MuYXBpLnYxLkF0dGFjaG1lbnRCA+BBAhI0Cgt1cGRhdGVfbWFzaxgCIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5GaWVsZE1hc2tCA+BBAiJIChdEZWxldGVBdHRhY2htZW50UmVxdWVzdBItCgRuYW1lGAEgASgJQh/gQQL6QRkKF21lbW9zLmFwaS52MS9BdHRhY2htZW50IjMKHUJhdGNoRGVsZXRlQXR0YWNobWVudHNSZXF1ZXN0EhIKBW5hbWVzGAEgAygJQgPgQQIqaAoRTW90aW9uTWVkaWFGYW1pbHkSIwofTU9USU9OX01FRElBX0ZBTUlMWV9VTlNQRUNJRklFRBAAEhQKEEFQUExFX0xJVkVfUEhPVE8QARIYChRBTkRST0lEX01PVElPTl9QSE9UTxACKlkKD01vdGlvbk1lZGlhUm9sZRIhCh1NT1RJT05fTUVESUFfUk9MRV9VTlNQRUNJRklFRBAAEgkKBVNUSUxMEAESCQoFVklERU8QAhINCglDT05UQUlORVIQAzLbBwoRQXR0YWNobWVudFNlcnZpY2USiQEKEENyZWF0ZUF0dGFjaG1lbnQSJS5tZW1vcy5hcGkudjEuQ3JlYXRlQXR0YWNobWVudFJlcXVlc3QaGC5tZW1vcy5hcGkudjEuQXR0YWNobWVudCI02kEKYXR0YWNobWVudILT5JMCIToKYXR0YWNobWVudCITL2FwaS92MS9hdHRhY2htZW50cxKIAQoQVXBsb2FkQXR0YWNobWVudBIlLm1lbW9zLmFwaS52MS5VcGxvYWRBdHRhY2htZW50UmVxdWVzdBomLm1lbW9zLmFwaS52MS5VcGxvYWRBdHRhY2htZW50UmVzcG9uc2UiJYLT5JMCHzoBKiIaL2FwaS92MS9hdHRhY2htZW50czp1cGxvYWQSewoPTGlzdEF0dGFjaG1lbnRzEiQubWVtb3MuYXBpLnYxLkxpc3RBdHRhY2htZW50c1JlcXVlc3QaJS5tZW1vcy5hcGkudjEuTGlzdEF0dGFjaG1lbnRzUmVzcG9uc2UiG4LT5JMCFRITL2FwaS92MS9hdHRhY2htZW50cxJ6Cg1HZXRBdHRhY2htZW50EiIubWVtb3MuYXBpLnYxLkdldEF0dGFjaG1lbnRSZXF1ZXN0GhgubWVtb3MuYXBpLnYxLkF0dGFjaG1lbnQiK9pBBG5hbWWC0+STAh4SHC9hcGkvdjEve25hbWU9YXR0YWNobWVudHMvKn0SqQEKEFVwZGF0ZUF0dGFjaG1lbnQSJS5tZW1vcy5hcGkudjEuVXBkYXRlQXR0YWNobWVudFJlcXVlc3QaGC5tZW1vcy5hcGkudjEuQXR0YWNobWVudCJU2kEWYXR0YWNobWVudCx1cGRhdGVfbWFza4LT5JMCNToKYXR0YWNobWVudDInL2FwaS92MS97YXR0YWNobWVudC5uYW1lPWF0dGFjaG1lbnRzLyp9En4KEERlbGV0ZUF0dGFjaG1lbnQSJS5tZW1vcy5hcGkudjEuRGVsZXRlQXR0YWNobWVudFJlcXVlc3QaFi5nb29nbGUucHJvdG9idWYuRW1wdHkiK9pBBG5hbWWC0+STAh4qHC9hcGkvdjEve25hbWU9YXR0YWNobWVudHMvKn0SiQEKFkJhdGNoRGVsZXRlQXR0YWNobWVudHMSKy5tZW1vcy5hcGkudjEuQmF0Y2hEZWxldGVBdHRhY2htZW50c1JlcXVlc3QaFi5nb29nbGUucHJvdG9idWYuRW1wdHkiKoLT5JMCJDoBKiIfL2FwaS92MS9hdHRhY2htZW50czpiYXRjaERlbGV0ZUKuAQoQY29tLm1lbW9zLmFwaS52MUIWQXR0YWNobWVudFNlcnZpY2VQcm90b1ABWjBnaXRodWIuY29tL3VzZW1lbW9zL21lbW9zL3Byb3RvL2dlbi9hcGkvdjE7YXBpdjGiAgNNQViqAgxNZW1vcy5BcGkuVjHKAgxNZW1vc1xBcGlcVjHiAhhNZW1vc1xBcGlcVjFcR1BCTWV0YWRhdGHqAg5NZW1vczo6QXBpOjpWMWIGcHJvdG8z", [file_google_api_annotations, file_google_api_client, file_google_api_field_behavior, file_google_api_resource, file_google_protobuf_empty, file_google_protobuf_field_mask, file_google_protobuf_timestamp]); /** * @generated from message memos.api.v1.MotionMedia @@ -359,6 +359,145 @@ export type CreateAttachmentRequest = Message<"memos.api.v1.CreateAttachmentRequ export const CreateAttachmentRequestSchema: GenMessage = /*@__PURE__*/ messageDesc(file_api_v1_attachment_service, 7); +/** + * @generated from message memos.api.v1.UploadAttachmentRequest + */ +export type UploadAttachmentRequest = Message<"memos.api.v1.UploadAttachmentRequest"> & { + /** + * Required. Start a new upload or continue an existing one. + * + * @generated from oneof memos.api.v1.UploadAttachmentRequest.upload + */ + upload: { + /** + * Starts a new upload. The same call may also carry data and finish_write. + * + * @generated from field: memos.api.v1.UploadAttachmentSpec spec = 1; + */ + value: UploadAttachmentSpec; + case: "spec"; + } | { + /** + * Continues the upload identified by a previous response. + * + * @generated from field: string upload_id = 2; + */ + value: string; + case: "uploadId"; + } | { case: undefined; value?: undefined }; + + /** + * Required. Zero-based byte offset at which data is written. Must equal the + * committed size, except when retrying the most recently accepted chunk + * with identical bytes and offset, which is accepted without writing again. + * + * @generated from field: int64 write_offset = 3; + */ + writeOffset: bigint; + + /** + * Optional. File bytes, at most max_chunk_size long. With no data and + * finish_write false, the call reports progress without writing, and + * write_offset is ignored. + * + * @generated from field: bytes data = 4; + */ + data: Uint8Array; + + /** + * Optional. Finalize the upload after writing data. The committed size must + * then equal total_size. Any later call for the same upload_id returns the + * created attachment. + * + * @generated from field: bool finish_write = 5; + */ + finishWrite: boolean; +}; + +/** + * Describes the message memos.api.v1.UploadAttachmentRequest. + * Use `create(UploadAttachmentRequestSchema)` to create a new message. + */ +export const UploadAttachmentRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_api_v1_attachment_service, 8); + +/** + * @generated from message memos.api.v1.UploadAttachmentSpec + */ +export type UploadAttachmentSpec = Message<"memos.api.v1.UploadAttachmentSpec"> & { + /** + * Required. Metadata for the attachment to create. content must be empty; + * file bytes are sent in data. + * + * @generated from field: memos.api.v1.Attachment attachment = 1; + */ + attachment?: Attachment | undefined; + + /** + * Optional. The attachment ID to use for this attachment. + * If empty, a unique ID will be generated. + * Format: ^[a-zA-Z0-9]([a-zA-Z0-9-]{0,34}[a-zA-Z0-9])?$ + * + * @generated from field: string attachment_id = 2; + */ + attachmentId: string; + + /** + * Optional. Total size of the file in bytes before media processing. + * Zero represents an empty file. + * + * @generated from field: int64 total_size = 3; + */ + totalSize: bigint; +}; + +/** + * Describes the message memos.api.v1.UploadAttachmentSpec. + * Use `create(UploadAttachmentSpecSchema)` to create a new message. + */ +export const UploadAttachmentSpecSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_api_v1_attachment_service, 9); + +/** + * @generated from message memos.api.v1.UploadAttachmentResponse + */ +export type UploadAttachmentResponse = Message<"memos.api.v1.UploadAttachmentResponse"> & { + /** + * Opaque ID for subsequent calls. This is not a resource name. + * + * @generated from field: string upload_id = 1; + */ + uploadId: string; + + /** + * Number of file bytes committed so far. + * + * @generated from field: int64 committed_size = 2; + */ + committedSize: bigint; + + /** + * Set once the upload has been finalized. + * + * @generated from field: memos.api.v1.Attachment attachment = 3; + */ + attachment?: Attachment | undefined; + + /** + * Maximum number of data bytes accepted in one call. + * + * @generated from field: int32 max_chunk_size = 4; + */ + maxChunkSize: number; +}; + +/** + * Describes the message memos.api.v1.UploadAttachmentResponse. + * Use `create(UploadAttachmentResponseSchema)` to create a new message. + */ +export const UploadAttachmentResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_api_v1_attachment_service, 10); + /** * @generated from message memos.api.v1.ListAttachmentsRequest */ @@ -409,7 +548,7 @@ export type ListAttachmentsRequest = Message<"memos.api.v1.ListAttachmentsReques * Use `create(ListAttachmentsRequestSchema)` to create a new message. */ export const ListAttachmentsRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_api_v1_attachment_service, 8); + messageDesc(file_api_v1_attachment_service, 11); /** * @generated from message memos.api.v1.ListAttachmentsResponse @@ -436,7 +575,7 @@ export type ListAttachmentsResponse = Message<"memos.api.v1.ListAttachmentsRespo * Use `create(ListAttachmentsResponseSchema)` to create a new message. */ export const ListAttachmentsResponseSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_api_v1_attachment_service, 9); + messageDesc(file_api_v1_attachment_service, 12); /** * @generated from message memos.api.v1.GetAttachmentRequest @@ -456,7 +595,7 @@ export type GetAttachmentRequest = Message<"memos.api.v1.GetAttachmentRequest"> * Use `create(GetAttachmentRequestSchema)` to create a new message. */ export const GetAttachmentRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_api_v1_attachment_service, 10); + messageDesc(file_api_v1_attachment_service, 13); /** * @generated from message memos.api.v1.UpdateAttachmentRequest @@ -482,7 +621,7 @@ export type UpdateAttachmentRequest = Message<"memos.api.v1.UpdateAttachmentRequ * Use `create(UpdateAttachmentRequestSchema)` to create a new message. */ export const UpdateAttachmentRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_api_v1_attachment_service, 11); + messageDesc(file_api_v1_attachment_service, 14); /** * @generated from message memos.api.v1.DeleteAttachmentRequest @@ -502,7 +641,7 @@ export type DeleteAttachmentRequest = Message<"memos.api.v1.DeleteAttachmentRequ * Use `create(DeleteAttachmentRequestSchema)` to create a new message. */ export const DeleteAttachmentRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_api_v1_attachment_service, 12); + messageDesc(file_api_v1_attachment_service, 15); /** * @generated from message memos.api.v1.BatchDeleteAttachmentsRequest @@ -519,7 +658,7 @@ export type BatchDeleteAttachmentsRequest = Message<"memos.api.v1.BatchDeleteAtt * Use `create(BatchDeleteAttachmentsRequestSchema)` to create a new message. */ export const BatchDeleteAttachmentsRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_api_v1_attachment_service, 13); + messageDesc(file_api_v1_attachment_service, 16); /** * @generated from enum memos.api.v1.MotionMediaFamily @@ -592,6 +731,19 @@ export const AttachmentService: GenService<{ input: typeof CreateAttachmentRequestSchema; output: typeof AttachmentSchema; }, + /** + * UploadAttachment uploads a file in bounded chunks. The first call carries + * the spec and returns an upload_id; later calls carry that upload_id. + * Uploads are bound to the authenticated user, expire after 30 minutes of + * inactivity, and do not survive a server restart. + * + * @generated from rpc memos.api.v1.AttachmentService.UploadAttachment + */ + uploadAttachment: { + methodKind: "unary"; + input: typeof UploadAttachmentRequestSchema; + output: typeof UploadAttachmentResponseSchema; + }, /** * ListAttachments lists all attachments. * diff --git a/web/tests/upload-media-metadata.test.ts b/web/tests/upload-media-metadata.test.ts index dd1af1e6..408c9b63 100644 --- a/web/tests/upload-media-metadata.test.ts +++ b/web/tests/upload-media-metadata.test.ts @@ -4,14 +4,14 @@ import type { LocalFile } from "@/components/MemoEditor/types/attachment"; import { MediaMetadataSchema } from "@/types/proto/api/v1/attachment_service_pb"; const mocks = vi.hoisted(() => ({ - createAttachment: vi.fn(), + uploadAttachment: vi.fn(), extractMetadata: vi.fn(), })); // The ingest helper lives beside useFileUpload, whose module graph reaches // AuthContext and the query hooks; stub every client they name-import. vi.mock("@/connect", () => ({ - attachmentServiceClient: { createAttachment: mocks.createAttachment }, + attachmentServiceClient: { uploadAttachment: mocks.uploadAttachment }, authServiceClient: {}, userServiceClient: {}, memoViewServiceClient: {}, @@ -58,14 +58,19 @@ describe("media metadata at file ingest", () => { describe("uploadService media metadata", () => { beforeEach(() => { - mocks.createAttachment.mockImplementation(async ({ attachment }) => attachment); + mocks.uploadAttachment.mockImplementation(async ({ data }) => ({ + uploadId: "test-id", + maxChunkSize: 2 * 1024 * 1024, + committedSize: BigInt(data?.length ?? 0), + attachment: data ? { name: "attachments/test" } : undefined, + })); }); it("submits no metadata for files ingested without any", async () => { await uploadService.uploadFiles([localImage()]); - expect(mocks.createAttachment).toHaveBeenCalledOnce(); - expect(mocks.createAttachment.mock.calls[0][0].attachment.mediaMetadata).toBeUndefined(); + expect(mocks.uploadAttachment).toHaveBeenCalledTimes(2); + expect(mocks.uploadAttachment.mock.calls[0][0].upload.value.attachment.mediaMetadata).toBeUndefined(); }); it("submits the metadata extracted at ingest", async () => { @@ -73,14 +78,14 @@ describe("uploadService media metadata", () => { await uploadService.uploadFiles([localImage(Promise.resolve(metadata))]); - expect(mocks.createAttachment.mock.calls[0][0].attachment.mediaMetadata).toEqual(metadata); + expect(mocks.uploadAttachment.mock.calls[0][0].upload.value.attachment.mediaMetadata).toEqual(metadata); }); it("continues without metadata when extraction produced no usable values", async () => { await uploadService.uploadFiles([localImage(Promise.resolve(undefined))]); - expect(mocks.createAttachment).toHaveBeenCalledOnce(); - expect(mocks.createAttachment.mock.calls[0][0].attachment.mediaMetadata).toBeUndefined(); + expect(mocks.uploadAttachment).toHaveBeenCalledTimes(2); + expect(mocks.uploadAttachment.mock.calls[0][0].upload.value.attachment.mediaMetadata).toBeUndefined(); }); it("submits ingest-time metadata end to end", async () => { @@ -90,6 +95,6 @@ describe("uploadService media metadata", () => { await uploadService.uploadFiles(toLocalFiles([file], { createBlobUrl, saveMediaMetadata: true })); - expect(mocks.createAttachment.mock.calls[0][0].attachment.mediaMetadata).toEqual(metadata); + expect(mocks.uploadAttachment.mock.calls[0][0].upload.value.attachment.mediaMetadata).toEqual(metadata); }); }); diff --git a/web/tests/upload-service.test.ts b/web/tests/upload-service.test.ts new file mode 100644 index 00000000..83ff9cb7 --- /dev/null +++ b/web/tests/upload-service.test.ts @@ -0,0 +1,98 @@ +import { create } from "@bufbuild/protobuf"; +import { Code, ConnectError } from "@connectrpc/connect"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { AttachmentSchema } from "@/types/proto/api/v1/attachment_service_pb"; + +const { uploadAttachment } = vi.hoisted(() => ({ uploadAttachment: vi.fn() })); +vi.mock("@/connect", () => ({ attachmentServiceClient: { uploadAttachment } })); + +import { uploadService } from "@/components/MemoEditor/services/uploadService"; + +const attachment = create(AttachmentSchema, { name: "attachments/uploaded" }); +const localFile = (content: string) => ({ + file: new File([content], "test.txt", { type: "text/plain" }), + origin: "upload" as const, + previewUrl: "blob:test", +}); + +describe("chunked attachment uploads", () => { + beforeEach(() => { + uploadAttachment.mockImplementation(async (request) => ({ + uploadId: "opaque-id", + maxChunkSize: 3, + committedSize: request.data ? request.writeOffset + BigInt(request.data.length) : 0n, + attachment: request.finishWrite ? attachment : undefined, + })); + }); + + it("reads only file slices and writes consecutive offsets", async () => { + const local = localFile("abcdefgh"); + const wholeFileRead = vi.spyOn(local.file, "arrayBuffer").mockRejectedValue(new Error("must not read the whole file")); + const slice = vi.spyOn(local.file, "slice"); + + await expect(uploadService.uploadFile(local)).resolves.toEqual(attachment); + + expect(wholeFileRead).not.toHaveBeenCalled(); + expect(slice.mock.calls).toEqual([ + [0, 3], + [3, 6], + [6, 8], + ]); + const requests = uploadAttachment.mock.calls.map(([request]) => request); + expect(requests[0].upload.case).toBe("spec"); + expect(requests[0].upload.value.totalSize).toBe(8n); + expect(requests[0].upload.value.attachment.content).toHaveLength(0); + expect(requests[0].data).toBeUndefined(); + expect(requests.slice(1).map((request) => request.upload)).toEqual(Array(3).fill({ case: "uploadId", value: "opaque-id" })); + expect(requests.slice(1).map((request) => request.writeOffset)).toEqual([0n, 3n, 6n]); + expect(requests.slice(1).map((request) => new TextDecoder().decode(request.data))).toEqual(["abc", "def", "gh"]); + expect(requests.slice(1).map((request) => request.finishWrite)).toEqual([false, false, true]); + }); + + it("finalizes an empty file", async () => { + await expect(uploadService.uploadFile(localFile(""))).resolves.toEqual(attachment); + expect(uploadAttachment).toHaveBeenCalledTimes(2); + expect(uploadAttachment.mock.calls[1][0]).toMatchObject({ writeOffset: 0n, finishWrite: true }); + expect(uploadAttachment.mock.calls[1][0].data).toHaveLength(0); + }); + + it("retries an identical final chunk after a lost response", async () => { + const original = uploadAttachment.getMockImplementation(); + let lost = false; + uploadAttachment.mockImplementation(async (request) => { + if (request.finishWrite && !lost) { + lost = true; + throw new ConnectError("response lost", Code.Unavailable); + } + return original?.(request); + }); + + await expect(uploadService.uploadFile(localFile("abc"))).resolves.toEqual(attachment); + expect(uploadAttachment.mock.calls[1][0]).toBe(uploadAttachment.mock.calls[2][0]); + }); + + it("does not retry permission errors", async () => { + uploadAttachment.mockRejectedValueOnce(new ConnectError("denied", Code.PermissionDenied)); + await expect(uploadService.uploadFile(localFile("abc"))).rejects.toMatchObject({ code: Code.PermissionDenied }); + expect(uploadAttachment).toHaveBeenCalledTimes(1); + }); + + it("bounds retries and stops on cancellation", async () => { + const initial = { uploadId: "id", committedSize: 0n, maxChunkSize: 3 }; + uploadAttachment.mockResolvedValueOnce(initial).mockRejectedValue(new ConnectError("offline", Code.Unavailable)); + await expect(uploadService.uploadFile(localFile("abc"))).rejects.toMatchObject({ code: Code.Unavailable }); + expect(uploadAttachment).toHaveBeenCalledTimes(4); + + uploadAttachment.mockReset().mockResolvedValue(initial); + const controller = new AbortController(); + controller.abort(); + await expect(uploadService.uploadFile(localFile("abc"), controller.signal)).rejects.toThrow(); + expect(uploadAttachment).toHaveBeenCalledTimes(1); + }); + + it("rejects invalid progress instead of looping or skipping bytes", async () => { + uploadAttachment.mockResolvedValue({ uploadId: "id", committedSize: 0n, maxChunkSize: 3 }); + await expect(uploadService.uploadFile(localFile("abcdef"))).rejects.toThrow("Unexpected upload offset"); + expect(uploadAttachment).toHaveBeenCalledTimes(2); + }); +});