feat(attachment): upload files in resumable chunks
Add UploadAttachment, a unary RPC that accepts a file in bounded chunks and streams it to a temp file instead of holding the whole blob in one request body and in memory. A call carrying a spec starts an upload and returns an opaque upload_id; later calls carry that id with write_offset, data, and finish_write, following the Cloud Storage WriteObject shape. Retrying the last chunk after a lost response is idempotent, uploads are bound to their owner, and they expire after 30 minutes of inactivity. CreateAttachment and the chunked finalize now share one processing pipeline for motion-photo detection, EXIF stripping, and storage. The motion-photo detector works over an io.ReaderAt so large JPEGs are never loaded whole, and the request body cap is a single per-procedure lookup. The web editor uploads through the new RPC in 2 MiB chunks with bounded concurrency under the server's per-user limit.
This commit is contained in:
parent
805e608aa5
commit
b3e67399b8
27 changed files with 2185 additions and 223 deletions
|
|
@ -20,25 +20,11 @@ var (
|
|||
|
||||
const maxMetadataScanBytes = 256 * 1024
|
||||
|
||||
// DetectJPEG detects embedded video in an in-memory JPEG.
|
||||
func DetectJPEG(blob []byte) *Detection {
|
||||
if len(blob) < 16 || !bytes.HasPrefix(blob, []byte{0xFF, 0xD8}) {
|
||||
return nil
|
||||
}
|
||||
|
||||
text := string(blob[:min(len(blob), maxMetadataScanBytes)])
|
||||
if !motionPhotoMarkerRegex.MatchString(text) {
|
||||
return nil
|
||||
}
|
||||
|
||||
videoStart := detectVideoStart(blob, text)
|
||||
if videoStart < 0 || videoStart >= len(blob) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &Detection{
|
||||
VideoStart: videoStart,
|
||||
PresentationTimestampUs: parsePresentationTimestampUs(text),
|
||||
}
|
||||
// bytes.Reader never fails within bounds, so the error is unreachable.
|
||||
detection, _ := DetectJPEGReader(bytes.NewReader(blob), int64(len(blob)))
|
||||
return detection
|
||||
}
|
||||
|
||||
func ExtractVideo(blob []byte) ([]byte, *Detection) {
|
||||
|
|
@ -55,19 +41,6 @@ func ExtractVideo(blob []byte) ([]byte, *Detection) {
|
|||
return videoBlob, detection
|
||||
}
|
||||
|
||||
func detectVideoStart(blob []byte, text string) int {
|
||||
if matches := microVideoOffsetRegex.FindStringSubmatch(text); len(matches) == 2 {
|
||||
if offset, err := strconv.Atoi(matches[1]); err == nil && offset > 0 && offset < len(blob) {
|
||||
start := len(blob) - offset
|
||||
if looksLikeMP4(blob[start:]) {
|
||||
return start
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return findEmbeddedMP4Start(blob)
|
||||
}
|
||||
|
||||
func parsePresentationTimestampUs(text string) int64 {
|
||||
matches := presentationRegex.FindStringSubmatch(text)
|
||||
if len(matches) != 2 {
|
||||
|
|
|
|||
56
internal/motionphoto/reader.go
Normal file
56
internal/motionphoto/reader.go
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
package motionphoto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// DetectJPEGReader detects embedded video without loading the JPEG into memory.
|
||||
// It reads at most the metadata header plus the blocks needed to locate the video.
|
||||
func DetectJPEGReader(reader io.ReaderAt, size int64) (*Detection, error) {
|
||||
if size < 16 {
|
||||
return nil, nil
|
||||
}
|
||||
header := make([]byte, min(size, maxMetadataScanBytes))
|
||||
if _, err := reader.ReadAt(header, 0); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !bytes.HasPrefix(header, []byte{0xFF, 0xD8}) || !motionPhotoMarkerRegex.Match(header) {
|
||||
return nil, nil
|
||||
}
|
||||
text := string(header)
|
||||
makeDetection := func(start int64) *Detection {
|
||||
return &Detection{VideoStart: int(start), PresentationTimestampUs: parsePresentationTimestampUs(text)}
|
||||
}
|
||||
if matches := microVideoOffsetRegex.FindStringSubmatch(text); len(matches) == 2 {
|
||||
if offset, err := strconv.ParseInt(matches[1], 10, 64); err == nil && offset >= 12 && offset < size {
|
||||
var probe [12]byte
|
||||
if _, err := reader.ReadAt(probe[:], size-offset); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if looksLikeMP4(probe[:]) {
|
||||
return makeDetection(size - offset), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to the last valid ftyp box, scanning backwards with overlap so
|
||||
// a box header crossing a block boundary is still found.
|
||||
buffer := make([]byte, 64*1024)
|
||||
for end := size; end >= 12; {
|
||||
start := max(int64(0), end-int64(len(buffer)))
|
||||
block := buffer[:end-start]
|
||||
if _, err := reader.ReadAt(block, start); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if index := findEmbeddedMP4Start(block); index >= 0 {
|
||||
return makeDetection(start + int64(index)), nil
|
||||
}
|
||||
if start == 0 {
|
||||
break
|
||||
}
|
||||
end = start + 11
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
45
internal/motionphoto/reader_test.go
Normal file
45
internal/motionphoto/reader_test.go
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
package motionphoto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/usememos/memos/internal/testutil"
|
||||
)
|
||||
|
||||
func TestDetectJPEGReader(t *testing.T) {
|
||||
for _, blob := range [][]byte{nil, []byte("not a JPEG"), []byte("\xff\xd8 no motion marker here at all")} {
|
||||
detected, err := DetectJPEGReader(bytes.NewReader(blob), int64(len(blob)))
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, detected)
|
||||
}
|
||||
fixture := testutil.BuildMotionPhotoJPEG()
|
||||
detected, err := DetectJPEGReader(bytes.NewReader(fixture), int64(len(fixture)))
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, detected)
|
||||
require.True(t, looksLikeMP4(fixture[detected.VideoStart:]))
|
||||
require.EqualValues(t, 123456, detected.PresentationTimestampUs)
|
||||
|
||||
// Exercise MP4 headers straddling both sides of the backwards scan boundary.
|
||||
for offset := -12; offset <= 12; offset++ {
|
||||
blob := make([]byte, 200_000)
|
||||
copy(blob, []byte("\xff\xd8 Camera:MotionPhoto=\"1\""))
|
||||
start := len(blob) - 64*1024 + offset
|
||||
copy(blob[start:], []byte{0, 0, 0, 16, 'f', 't', 'y', 'p', 'm', 'p', '4', '2'})
|
||||
detected, err := DetectJPEGReader(bytes.NewReader(blob), int64(len(blob)))
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, detected, "offset %d", offset)
|
||||
require.Equal(t, start, detected.VideoStart, "offset %d", offset)
|
||||
}
|
||||
|
||||
// Prefer the explicit MicroVideoOffset even when a later ftyp box exists.
|
||||
blob := make([]byte, 1024)
|
||||
copy(blob, []byte("\xff\xd8 Camera:MotionPhoto=\"1\" Camera:MicroVideoOffset=\"128\""))
|
||||
copy(blob[896:], []byte{0, 0, 0, 16, 'f', 't', 'y', 'p', 'm', 'p', '4', '2'})
|
||||
copy(blob[1000:], []byte{0, 0, 0, 16, 'f', 't', 'y', 'p', 'm', 'p', '4', '2'})
|
||||
detected, err = DetectJPEGReader(bytes.NewReader(blob), int64(len(blob)))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 896, detected.VideoStart)
|
||||
}
|
||||
|
|
@ -21,6 +21,16 @@ service AttachmentService {
|
|||
};
|
||||
option (google.api.method_signature) = "attachment";
|
||||
}
|
||||
// UploadAttachment uploads a file in bounded chunks. The first call carries
|
||||
// the spec and returns an upload_id; later calls carry that upload_id.
|
||||
// Uploads are bound to the authenticated user, expire after 30 minutes of
|
||||
// inactivity, and do not survive a server restart.
|
||||
rpc UploadAttachment(UploadAttachmentRequest) returns (UploadAttachmentResponse) {
|
||||
option (google.api.http) = {
|
||||
post: "/api/v1/attachments:upload"
|
||||
body: "*"
|
||||
};
|
||||
}
|
||||
// ListAttachments lists all attachments.
|
||||
rpc ListAttachments(ListAttachmentsRequest) returns (ListAttachmentsResponse) {
|
||||
option (google.api.http) = {get: "/api/v1/attachments"};
|
||||
|
|
@ -179,6 +189,61 @@ message CreateAttachmentRequest {
|
|||
string attachment_id = 2 [(google.api.field_behavior) = OPTIONAL];
|
||||
}
|
||||
|
||||
message UploadAttachmentRequest {
|
||||
// Required. Start a new upload or continue an existing one.
|
||||
oneof upload {
|
||||
// Starts a new upload. The same call may also carry data and finish_write.
|
||||
UploadAttachmentSpec spec = 1;
|
||||
|
||||
// Continues the upload identified by a previous response.
|
||||
string upload_id = 2;
|
||||
}
|
||||
|
||||
// Required. Zero-based byte offset at which data is written. Must equal the
|
||||
// committed size, except when retrying the most recently accepted chunk
|
||||
// with identical bytes and offset, which is accepted without writing again.
|
||||
int64 write_offset = 3 [(google.api.field_behavior) = REQUIRED];
|
||||
|
||||
// Optional. File bytes, at most max_chunk_size long. With no data and
|
||||
// finish_write false, the call reports progress without writing, and
|
||||
// write_offset is ignored.
|
||||
bytes data = 4 [(google.api.field_behavior) = OPTIONAL];
|
||||
|
||||
// Optional. Finalize the upload after writing data. The committed size must
|
||||
// then equal total_size. Any later call for the same upload_id returns the
|
||||
// created attachment.
|
||||
bool finish_write = 5 [(google.api.field_behavior) = OPTIONAL];
|
||||
}
|
||||
|
||||
message UploadAttachmentSpec {
|
||||
// Required. Metadata for the attachment to create. content must be empty;
|
||||
// file bytes are sent in data.
|
||||
Attachment attachment = 1 [(google.api.field_behavior) = REQUIRED];
|
||||
|
||||
// Optional. The attachment ID to use for this attachment.
|
||||
// If empty, a unique ID will be generated.
|
||||
// Format: ^[a-zA-Z0-9]([a-zA-Z0-9-]{0,34}[a-zA-Z0-9])?$
|
||||
string attachment_id = 2 [(google.api.field_behavior) = OPTIONAL];
|
||||
|
||||
// Optional. Total size of the file in bytes before media processing.
|
||||
// Zero represents an empty file.
|
||||
int64 total_size = 3 [(google.api.field_behavior) = OPTIONAL];
|
||||
}
|
||||
|
||||
message UploadAttachmentResponse {
|
||||
// Opaque ID for subsequent calls. This is not a resource name.
|
||||
string upload_id = 1;
|
||||
|
||||
// Number of file bytes committed so far.
|
||||
int64 committed_size = 2;
|
||||
|
||||
// Set once the upload has been finalized.
|
||||
Attachment attachment = 3;
|
||||
|
||||
// Maximum number of data bytes accepted in one call.
|
||||
int32 max_chunk_size = 4;
|
||||
}
|
||||
|
||||
message ListAttachmentsRequest {
|
||||
// Optional. The maximum number of attachments to return.
|
||||
// The service may return fewer than this value.
|
||||
|
|
|
|||
|
|
@ -37,6 +37,9 @@ const (
|
|||
// AttachmentServiceCreateAttachmentProcedure is the fully-qualified name of the AttachmentService's
|
||||
// CreateAttachment RPC.
|
||||
AttachmentServiceCreateAttachmentProcedure = "/memos.api.v1.AttachmentService/CreateAttachment"
|
||||
// AttachmentServiceUploadAttachmentProcedure is the fully-qualified name of the AttachmentService's
|
||||
// UploadAttachment RPC.
|
||||
AttachmentServiceUploadAttachmentProcedure = "/memos.api.v1.AttachmentService/UploadAttachment"
|
||||
// AttachmentServiceListAttachmentsProcedure is the fully-qualified name of the AttachmentService's
|
||||
// ListAttachments RPC.
|
||||
AttachmentServiceListAttachmentsProcedure = "/memos.api.v1.AttachmentService/ListAttachments"
|
||||
|
|
@ -58,6 +61,11 @@ const (
|
|||
type AttachmentServiceClient interface {
|
||||
// CreateAttachment creates a new attachment.
|
||||
CreateAttachment(context.Context, *connect.Request[v1.CreateAttachmentRequest]) (*connect.Response[v1.Attachment], error)
|
||||
// UploadAttachment uploads a file in bounded chunks. The first call carries
|
||||
// the spec and returns an upload_id; later calls carry that upload_id.
|
||||
// Uploads are bound to the authenticated user, expire after 30 minutes of
|
||||
// inactivity, and do not survive a server restart.
|
||||
UploadAttachment(context.Context, *connect.Request[v1.UploadAttachmentRequest]) (*connect.Response[v1.UploadAttachmentResponse], error)
|
||||
// ListAttachments lists all attachments.
|
||||
ListAttachments(context.Context, *connect.Request[v1.ListAttachmentsRequest]) (*connect.Response[v1.ListAttachmentsResponse], error)
|
||||
// GetAttachment returns an attachment by name.
|
||||
|
|
@ -87,6 +95,12 @@ func NewAttachmentServiceClient(httpClient connect.HTTPClient, baseURL string, o
|
|||
connect.WithSchema(attachmentServiceMethods.ByName("CreateAttachment")),
|
||||
connect.WithClientOptions(opts...),
|
||||
),
|
||||
uploadAttachment: connect.NewClient[v1.UploadAttachmentRequest, v1.UploadAttachmentResponse](
|
||||
httpClient,
|
||||
baseURL+AttachmentServiceUploadAttachmentProcedure,
|
||||
connect.WithSchema(attachmentServiceMethods.ByName("UploadAttachment")),
|
||||
connect.WithClientOptions(opts...),
|
||||
),
|
||||
listAttachments: connect.NewClient[v1.ListAttachmentsRequest, v1.ListAttachmentsResponse](
|
||||
httpClient,
|
||||
baseURL+AttachmentServiceListAttachmentsProcedure,
|
||||
|
|
@ -123,6 +137,7 @@ func NewAttachmentServiceClient(httpClient connect.HTTPClient, baseURL string, o
|
|||
// attachmentServiceClient implements AttachmentServiceClient.
|
||||
type attachmentServiceClient struct {
|
||||
createAttachment *connect.Client[v1.CreateAttachmentRequest, v1.Attachment]
|
||||
uploadAttachment *connect.Client[v1.UploadAttachmentRequest, v1.UploadAttachmentResponse]
|
||||
listAttachments *connect.Client[v1.ListAttachmentsRequest, v1.ListAttachmentsResponse]
|
||||
getAttachment *connect.Client[v1.GetAttachmentRequest, v1.Attachment]
|
||||
updateAttachment *connect.Client[v1.UpdateAttachmentRequest, v1.Attachment]
|
||||
|
|
@ -135,6 +150,11 @@ func (c *attachmentServiceClient) CreateAttachment(ctx context.Context, req *con
|
|||
return c.createAttachment.CallUnary(ctx, req)
|
||||
}
|
||||
|
||||
// UploadAttachment calls memos.api.v1.AttachmentService.UploadAttachment.
|
||||
func (c *attachmentServiceClient) UploadAttachment(ctx context.Context, req *connect.Request[v1.UploadAttachmentRequest]) (*connect.Response[v1.UploadAttachmentResponse], error) {
|
||||
return c.uploadAttachment.CallUnary(ctx, req)
|
||||
}
|
||||
|
||||
// ListAttachments calls memos.api.v1.AttachmentService.ListAttachments.
|
||||
func (c *attachmentServiceClient) ListAttachments(ctx context.Context, req *connect.Request[v1.ListAttachmentsRequest]) (*connect.Response[v1.ListAttachmentsResponse], error) {
|
||||
return c.listAttachments.CallUnary(ctx, req)
|
||||
|
|
@ -164,6 +184,11 @@ func (c *attachmentServiceClient) BatchDeleteAttachments(ctx context.Context, re
|
|||
type AttachmentServiceHandler interface {
|
||||
// CreateAttachment creates a new attachment.
|
||||
CreateAttachment(context.Context, *connect.Request[v1.CreateAttachmentRequest]) (*connect.Response[v1.Attachment], error)
|
||||
// UploadAttachment uploads a file in bounded chunks. The first call carries
|
||||
// the spec and returns an upload_id; later calls carry that upload_id.
|
||||
// Uploads are bound to the authenticated user, expire after 30 minutes of
|
||||
// inactivity, and do not survive a server restart.
|
||||
UploadAttachment(context.Context, *connect.Request[v1.UploadAttachmentRequest]) (*connect.Response[v1.UploadAttachmentResponse], error)
|
||||
// ListAttachments lists all attachments.
|
||||
ListAttachments(context.Context, *connect.Request[v1.ListAttachmentsRequest]) (*connect.Response[v1.ListAttachmentsResponse], error)
|
||||
// GetAttachment returns an attachment by name.
|
||||
|
|
@ -189,6 +214,12 @@ func NewAttachmentServiceHandler(svc AttachmentServiceHandler, opts ...connect.H
|
|||
connect.WithSchema(attachmentServiceMethods.ByName("CreateAttachment")),
|
||||
connect.WithHandlerOptions(opts...),
|
||||
)
|
||||
attachmentServiceUploadAttachmentHandler := connect.NewUnaryHandler(
|
||||
AttachmentServiceUploadAttachmentProcedure,
|
||||
svc.UploadAttachment,
|
||||
connect.WithSchema(attachmentServiceMethods.ByName("UploadAttachment")),
|
||||
connect.WithHandlerOptions(opts...),
|
||||
)
|
||||
attachmentServiceListAttachmentsHandler := connect.NewUnaryHandler(
|
||||
AttachmentServiceListAttachmentsProcedure,
|
||||
svc.ListAttachments,
|
||||
|
|
@ -223,6 +254,8 @@ func NewAttachmentServiceHandler(svc AttachmentServiceHandler, opts ...connect.H
|
|||
switch r.URL.Path {
|
||||
case AttachmentServiceCreateAttachmentProcedure:
|
||||
attachmentServiceCreateAttachmentHandler.ServeHTTP(w, r)
|
||||
case AttachmentServiceUploadAttachmentProcedure:
|
||||
attachmentServiceUploadAttachmentHandler.ServeHTTP(w, r)
|
||||
case AttachmentServiceListAttachmentsProcedure:
|
||||
attachmentServiceListAttachmentsHandler.ServeHTTP(w, r)
|
||||
case AttachmentServiceGetAttachmentProcedure:
|
||||
|
|
@ -246,6 +279,10 @@ func (UnimplementedAttachmentServiceHandler) CreateAttachment(context.Context, *
|
|||
return nil, connect.NewError(connect.CodeUnimplemented, errors.New("memos.api.v1.AttachmentService.CreateAttachment is not implemented"))
|
||||
}
|
||||
|
||||
func (UnimplementedAttachmentServiceHandler) UploadAttachment(context.Context, *connect.Request[v1.UploadAttachmentRequest]) (*connect.Response[v1.UploadAttachmentResponse], error) {
|
||||
return nil, connect.NewError(connect.CodeUnimplemented, errors.New("memos.api.v1.AttachmentService.UploadAttachment is not implemented"))
|
||||
}
|
||||
|
||||
func (UnimplementedAttachmentServiceHandler) ListAttachments(context.Context, *connect.Request[v1.ListAttachmentsRequest]) (*connect.Response[v1.ListAttachmentsResponse], error) {
|
||||
return nil, connect.NewError(connect.CodeUnimplemented, errors.New("memos.api.v1.AttachmentService.ListAttachments is not implemented"))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -770,6 +770,264 @@ func (x *CreateAttachmentRequest) GetAttachmentId() string {
|
|||
return ""
|
||||
}
|
||||
|
||||
type UploadAttachmentRequest struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
// Required. Start a new upload or continue an existing one.
|
||||
//
|
||||
// Types that are valid to be assigned to Upload:
|
||||
//
|
||||
// *UploadAttachmentRequest_Spec
|
||||
// *UploadAttachmentRequest_UploadId
|
||||
Upload isUploadAttachmentRequest_Upload `protobuf_oneof:"upload"`
|
||||
// Required. Zero-based byte offset at which data is written. Must equal the
|
||||
// committed size, except when retrying the most recently accepted chunk
|
||||
// with identical bytes and offset, which is accepted without writing again.
|
||||
WriteOffset int64 `protobuf:"varint,3,opt,name=write_offset,json=writeOffset,proto3" json:"write_offset,omitempty"`
|
||||
// Optional. File bytes, at most max_chunk_size long. With no data and
|
||||
// finish_write false, the call reports progress without writing, and
|
||||
// write_offset is ignored.
|
||||
Data []byte `protobuf:"bytes,4,opt,name=data,proto3" json:"data,omitempty"`
|
||||
// Optional. Finalize the upload after writing data. The committed size must
|
||||
// then equal total_size. Any later call for the same upload_id returns the
|
||||
// created attachment.
|
||||
FinishWrite bool `protobuf:"varint,5,opt,name=finish_write,json=finishWrite,proto3" json:"finish_write,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *UploadAttachmentRequest) Reset() {
|
||||
*x = UploadAttachmentRequest{}
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[8]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *UploadAttachmentRequest) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*UploadAttachmentRequest) ProtoMessage() {}
|
||||
|
||||
func (x *UploadAttachmentRequest) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[8]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use UploadAttachmentRequest.ProtoReflect.Descriptor instead.
|
||||
func (*UploadAttachmentRequest) Descriptor() ([]byte, []int) {
|
||||
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{8}
|
||||
}
|
||||
|
||||
func (x *UploadAttachmentRequest) GetUpload() isUploadAttachmentRequest_Upload {
|
||||
if x != nil {
|
||||
return x.Upload
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *UploadAttachmentRequest) GetSpec() *UploadAttachmentSpec {
|
||||
if x != nil {
|
||||
if x, ok := x.Upload.(*UploadAttachmentRequest_Spec); ok {
|
||||
return x.Spec
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *UploadAttachmentRequest) GetUploadId() string {
|
||||
if x != nil {
|
||||
if x, ok := x.Upload.(*UploadAttachmentRequest_UploadId); ok {
|
||||
return x.UploadId
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *UploadAttachmentRequest) GetWriteOffset() int64 {
|
||||
if x != nil {
|
||||
return x.WriteOffset
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *UploadAttachmentRequest) GetData() []byte {
|
||||
if x != nil {
|
||||
return x.Data
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *UploadAttachmentRequest) GetFinishWrite() bool {
|
||||
if x != nil {
|
||||
return x.FinishWrite
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type isUploadAttachmentRequest_Upload interface {
|
||||
isUploadAttachmentRequest_Upload()
|
||||
}
|
||||
|
||||
type UploadAttachmentRequest_Spec struct {
|
||||
// Starts a new upload. The same call may also carry data and finish_write.
|
||||
Spec *UploadAttachmentSpec `protobuf:"bytes,1,opt,name=spec,proto3,oneof"`
|
||||
}
|
||||
|
||||
type UploadAttachmentRequest_UploadId struct {
|
||||
// Continues the upload identified by a previous response.
|
||||
UploadId string `protobuf:"bytes,2,opt,name=upload_id,json=uploadId,proto3,oneof"`
|
||||
}
|
||||
|
||||
func (*UploadAttachmentRequest_Spec) isUploadAttachmentRequest_Upload() {}
|
||||
|
||||
func (*UploadAttachmentRequest_UploadId) isUploadAttachmentRequest_Upload() {}
|
||||
|
||||
type UploadAttachmentSpec struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
// Required. Metadata for the attachment to create. content must be empty;
|
||||
// file bytes are sent in data.
|
||||
Attachment *Attachment `protobuf:"bytes,1,opt,name=attachment,proto3" json:"attachment,omitempty"`
|
||||
// Optional. The attachment ID to use for this attachment.
|
||||
// If empty, a unique ID will be generated.
|
||||
// Format: ^[a-zA-Z0-9]([a-zA-Z0-9-]{0,34}[a-zA-Z0-9])?$
|
||||
AttachmentId string `protobuf:"bytes,2,opt,name=attachment_id,json=attachmentId,proto3" json:"attachment_id,omitempty"`
|
||||
// Optional. Total size of the file in bytes before media processing.
|
||||
// Zero represents an empty file.
|
||||
TotalSize int64 `protobuf:"varint,3,opt,name=total_size,json=totalSize,proto3" json:"total_size,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *UploadAttachmentSpec) Reset() {
|
||||
*x = UploadAttachmentSpec{}
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[9]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *UploadAttachmentSpec) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*UploadAttachmentSpec) ProtoMessage() {}
|
||||
|
||||
func (x *UploadAttachmentSpec) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[9]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use UploadAttachmentSpec.ProtoReflect.Descriptor instead.
|
||||
func (*UploadAttachmentSpec) Descriptor() ([]byte, []int) {
|
||||
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{9}
|
||||
}
|
||||
|
||||
func (x *UploadAttachmentSpec) GetAttachment() *Attachment {
|
||||
if x != nil {
|
||||
return x.Attachment
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *UploadAttachmentSpec) GetAttachmentId() string {
|
||||
if x != nil {
|
||||
return x.AttachmentId
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *UploadAttachmentSpec) GetTotalSize() int64 {
|
||||
if x != nil {
|
||||
return x.TotalSize
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type UploadAttachmentResponse struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
// Opaque ID for subsequent calls. This is not a resource name.
|
||||
UploadId string `protobuf:"bytes,1,opt,name=upload_id,json=uploadId,proto3" json:"upload_id,omitempty"`
|
||||
// Number of file bytes committed so far.
|
||||
CommittedSize int64 `protobuf:"varint,2,opt,name=committed_size,json=committedSize,proto3" json:"committed_size,omitempty"`
|
||||
// Set once the upload has been finalized.
|
||||
Attachment *Attachment `protobuf:"bytes,3,opt,name=attachment,proto3" json:"attachment,omitempty"`
|
||||
// Maximum number of data bytes accepted in one call.
|
||||
MaxChunkSize int32 `protobuf:"varint,4,opt,name=max_chunk_size,json=maxChunkSize,proto3" json:"max_chunk_size,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *UploadAttachmentResponse) Reset() {
|
||||
*x = UploadAttachmentResponse{}
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[10]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *UploadAttachmentResponse) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*UploadAttachmentResponse) ProtoMessage() {}
|
||||
|
||||
func (x *UploadAttachmentResponse) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[10]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use UploadAttachmentResponse.ProtoReflect.Descriptor instead.
|
||||
func (*UploadAttachmentResponse) Descriptor() ([]byte, []int) {
|
||||
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{10}
|
||||
}
|
||||
|
||||
func (x *UploadAttachmentResponse) GetUploadId() string {
|
||||
if x != nil {
|
||||
return x.UploadId
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *UploadAttachmentResponse) GetCommittedSize() int64 {
|
||||
if x != nil {
|
||||
return x.CommittedSize
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *UploadAttachmentResponse) GetAttachment() *Attachment {
|
||||
if x != nil {
|
||||
return x.Attachment
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *UploadAttachmentResponse) GetMaxChunkSize() int32 {
|
||||
if x != nil {
|
||||
return x.MaxChunkSize
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type ListAttachmentsRequest struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
// Optional. The maximum number of attachments to return.
|
||||
|
|
@ -798,7 +1056,7 @@ type ListAttachmentsRequest struct {
|
|||
|
||||
func (x *ListAttachmentsRequest) Reset() {
|
||||
*x = ListAttachmentsRequest{}
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[8]
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[11]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
|
@ -810,7 +1068,7 @@ func (x *ListAttachmentsRequest) String() string {
|
|||
func (*ListAttachmentsRequest) ProtoMessage() {}
|
||||
|
||||
func (x *ListAttachmentsRequest) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[8]
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[11]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
|
|
@ -823,7 +1081,7 @@ func (x *ListAttachmentsRequest) ProtoReflect() protoreflect.Message {
|
|||
|
||||
// Deprecated: Use ListAttachmentsRequest.ProtoReflect.Descriptor instead.
|
||||
func (*ListAttachmentsRequest) Descriptor() ([]byte, []int) {
|
||||
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{8}
|
||||
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{11}
|
||||
}
|
||||
|
||||
func (x *ListAttachmentsRequest) GetPageSize() int32 {
|
||||
|
|
@ -867,7 +1125,7 @@ type ListAttachmentsResponse struct {
|
|||
|
||||
func (x *ListAttachmentsResponse) Reset() {
|
||||
*x = ListAttachmentsResponse{}
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[9]
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[12]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
|
@ -879,7 +1137,7 @@ func (x *ListAttachmentsResponse) String() string {
|
|||
func (*ListAttachmentsResponse) ProtoMessage() {}
|
||||
|
||||
func (x *ListAttachmentsResponse) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[9]
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[12]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
|
|
@ -892,7 +1150,7 @@ func (x *ListAttachmentsResponse) ProtoReflect() protoreflect.Message {
|
|||
|
||||
// Deprecated: Use ListAttachmentsResponse.ProtoReflect.Descriptor instead.
|
||||
func (*ListAttachmentsResponse) Descriptor() ([]byte, []int) {
|
||||
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{9}
|
||||
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{12}
|
||||
}
|
||||
|
||||
func (x *ListAttachmentsResponse) GetAttachments() []*Attachment {
|
||||
|
|
@ -920,7 +1178,7 @@ type GetAttachmentRequest struct {
|
|||
|
||||
func (x *GetAttachmentRequest) Reset() {
|
||||
*x = GetAttachmentRequest{}
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[10]
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[13]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
|
@ -932,7 +1190,7 @@ func (x *GetAttachmentRequest) String() string {
|
|||
func (*GetAttachmentRequest) ProtoMessage() {}
|
||||
|
||||
func (x *GetAttachmentRequest) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[10]
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[13]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
|
|
@ -945,7 +1203,7 @@ func (x *GetAttachmentRequest) ProtoReflect() protoreflect.Message {
|
|||
|
||||
// Deprecated: Use GetAttachmentRequest.ProtoReflect.Descriptor instead.
|
||||
func (*GetAttachmentRequest) Descriptor() ([]byte, []int) {
|
||||
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{10}
|
||||
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{13}
|
||||
}
|
||||
|
||||
func (x *GetAttachmentRequest) GetName() string {
|
||||
|
|
@ -967,7 +1225,7 @@ type UpdateAttachmentRequest struct {
|
|||
|
||||
func (x *UpdateAttachmentRequest) Reset() {
|
||||
*x = UpdateAttachmentRequest{}
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[11]
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[14]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
|
@ -979,7 +1237,7 @@ func (x *UpdateAttachmentRequest) String() string {
|
|||
func (*UpdateAttachmentRequest) ProtoMessage() {}
|
||||
|
||||
func (x *UpdateAttachmentRequest) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[11]
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[14]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
|
|
@ -992,7 +1250,7 @@ func (x *UpdateAttachmentRequest) ProtoReflect() protoreflect.Message {
|
|||
|
||||
// Deprecated: Use UpdateAttachmentRequest.ProtoReflect.Descriptor instead.
|
||||
func (*UpdateAttachmentRequest) Descriptor() ([]byte, []int) {
|
||||
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{11}
|
||||
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{14}
|
||||
}
|
||||
|
||||
func (x *UpdateAttachmentRequest) GetAttachment() *Attachment {
|
||||
|
|
@ -1020,7 +1278,7 @@ type DeleteAttachmentRequest struct {
|
|||
|
||||
func (x *DeleteAttachmentRequest) Reset() {
|
||||
*x = DeleteAttachmentRequest{}
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[12]
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[15]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
|
@ -1032,7 +1290,7 @@ func (x *DeleteAttachmentRequest) String() string {
|
|||
func (*DeleteAttachmentRequest) ProtoMessage() {}
|
||||
|
||||
func (x *DeleteAttachmentRequest) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[12]
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[15]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
|
|
@ -1045,7 +1303,7 @@ func (x *DeleteAttachmentRequest) ProtoReflect() protoreflect.Message {
|
|||
|
||||
// Deprecated: Use DeleteAttachmentRequest.ProtoReflect.Descriptor instead.
|
||||
func (*DeleteAttachmentRequest) Descriptor() ([]byte, []int) {
|
||||
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{12}
|
||||
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{15}
|
||||
}
|
||||
|
||||
func (x *DeleteAttachmentRequest) GetName() string {
|
||||
|
|
@ -1064,7 +1322,7 @@ type BatchDeleteAttachmentsRequest struct {
|
|||
|
||||
func (x *BatchDeleteAttachmentsRequest) Reset() {
|
||||
*x = BatchDeleteAttachmentsRequest{}
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[13]
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[16]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
|
@ -1076,7 +1334,7 @@ func (x *BatchDeleteAttachmentsRequest) String() string {
|
|||
func (*BatchDeleteAttachmentsRequest) ProtoMessage() {}
|
||||
|
||||
func (x *BatchDeleteAttachmentsRequest) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[13]
|
||||
mi := &file_api_v1_attachment_service_proto_msgTypes[16]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
|
|
@ -1089,7 +1347,7 @@ func (x *BatchDeleteAttachmentsRequest) ProtoReflect() protoreflect.Message {
|
|||
|
||||
// Deprecated: Use BatchDeleteAttachmentsRequest.ProtoReflect.Descriptor instead.
|
||||
func (*BatchDeleteAttachmentsRequest) Descriptor() ([]byte, []int) {
|
||||
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{13}
|
||||
return file_api_v1_attachment_service_proto_rawDescGZIP(), []int{16}
|
||||
}
|
||||
|
||||
func (x *BatchDeleteAttachmentsRequest) GetNames() []string {
|
||||
|
|
@ -1174,7 +1432,28 @@ const file_api_v1_attachment_service_proto_rawDesc = "" +
|
|||
"\n" +
|
||||
"attachment\x18\x01 \x01(\v2\x18.memos.api.v1.AttachmentB\x03\xe0A\x02R\n" +
|
||||
"attachment\x12(\n" +
|
||||
"\rattachment_id\x18\x02 \x01(\tB\x03\xe0A\x01R\fattachmentId\"\xba\x01\n" +
|
||||
"\rattachment_id\x18\x02 \x01(\tB\x03\xe0A\x01R\fattachmentId\"\xe5\x01\n" +
|
||||
"\x17UploadAttachmentRequest\x128\n" +
|
||||
"\x04spec\x18\x01 \x01(\v2\".memos.api.v1.UploadAttachmentSpecH\x00R\x04spec\x12\x1d\n" +
|
||||
"\tupload_id\x18\x02 \x01(\tH\x00R\buploadId\x12&\n" +
|
||||
"\fwrite_offset\x18\x03 \x01(\x03B\x03\xe0A\x02R\vwriteOffset\x12\x17\n" +
|
||||
"\x04data\x18\x04 \x01(\fB\x03\xe0A\x01R\x04data\x12&\n" +
|
||||
"\ffinish_write\x18\x05 \x01(\bB\x03\xe0A\x01R\vfinishWriteB\b\n" +
|
||||
"\x06upload\"\xa3\x01\n" +
|
||||
"\x14UploadAttachmentSpec\x12=\n" +
|
||||
"\n" +
|
||||
"attachment\x18\x01 \x01(\v2\x18.memos.api.v1.AttachmentB\x03\xe0A\x02R\n" +
|
||||
"attachment\x12(\n" +
|
||||
"\rattachment_id\x18\x02 \x01(\tB\x03\xe0A\x01R\fattachmentId\x12\"\n" +
|
||||
"\n" +
|
||||
"total_size\x18\x03 \x01(\x03B\x03\xe0A\x01R\ttotalSize\"\xbe\x01\n" +
|
||||
"\x18UploadAttachmentResponse\x12\x1b\n" +
|
||||
"\tupload_id\x18\x01 \x01(\tR\buploadId\x12%\n" +
|
||||
"\x0ecommitted_size\x18\x02 \x01(\x03R\rcommittedSize\x128\n" +
|
||||
"\n" +
|
||||
"attachment\x18\x03 \x01(\v2\x18.memos.api.v1.AttachmentR\n" +
|
||||
"attachment\x12$\n" +
|
||||
"\x0emax_chunk_size\x18\x04 \x01(\x05R\fmaxChunkSize\"\xba\x01\n" +
|
||||
"\x16ListAttachmentsRequest\x12 \n" +
|
||||
"\tpage_size\x18\x01 \x01(\x05B\x03\xe0A\x01R\bpageSize\x12\"\n" +
|
||||
"\n" +
|
||||
|
|
@ -1207,11 +1486,12 @@ const file_api_v1_attachment_service_proto_rawDesc = "" +
|
|||
"\x1dMOTION_MEDIA_ROLE_UNSPECIFIED\x10\x00\x12\t\n" +
|
||||
"\x05STILL\x10\x01\x12\t\n" +
|
||||
"\x05VIDEO\x10\x02\x12\r\n" +
|
||||
"\tCONTAINER\x10\x032\xd0\x06\n" +
|
||||
"\tCONTAINER\x10\x032\xdb\a\n" +
|
||||
"\x11AttachmentService\x12\x89\x01\n" +
|
||||
"\x10CreateAttachment\x12%.memos.api.v1.CreateAttachmentRequest\x1a\x18.memos.api.v1.Attachment\"4\xdaA\n" +
|
||||
"attachment\x82\xd3\xe4\x93\x02!:\n" +
|
||||
"attachment\"\x13/api/v1/attachments\x12{\n" +
|
||||
"attachment\"\x13/api/v1/attachments\x12\x88\x01\n" +
|
||||
"\x10UploadAttachment\x12%.memos.api.v1.UploadAttachmentRequest\x1a&.memos.api.v1.UploadAttachmentResponse\"%\x82\xd3\xe4\x93\x02\x1f:\x01*\"\x1a/api/v1/attachments:upload\x12{\n" +
|
||||
"\x0fListAttachments\x12$.memos.api.v1.ListAttachmentsRequest\x1a%.memos.api.v1.ListAttachmentsResponse\"\x1b\x82\xd3\xe4\x93\x02\x15\x12\x13/api/v1/attachments\x12z\n" +
|
||||
"\rGetAttachment\x12\".memos.api.v1.GetAttachmentRequest\x1a\x18.memos.api.v1.Attachment\"+\xdaA\x04name\x82\xd3\xe4\x93\x02\x1e\x12\x1c/api/v1/{name=attachments/*}\x12\xa9\x01\n" +
|
||||
"\x10UpdateAttachment\x12%.memos.api.v1.UpdateAttachmentRequest\x1a\x18.memos.api.v1.Attachment\"T\xdaA\x16attachment,update_mask\x82\xd3\xe4\x93\x025:\n" +
|
||||
|
|
@ -1233,7 +1513,7 @@ func file_api_v1_attachment_service_proto_rawDescGZIP() []byte {
|
|||
}
|
||||
|
||||
var file_api_v1_attachment_service_proto_enumTypes = make([]protoimpl.EnumInfo, 2)
|
||||
var file_api_v1_attachment_service_proto_msgTypes = make([]protoimpl.MessageInfo, 14)
|
||||
var file_api_v1_attachment_service_proto_msgTypes = make([]protoimpl.MessageInfo, 17)
|
||||
var file_api_v1_attachment_service_proto_goTypes = []any{
|
||||
(MotionMediaFamily)(0), // 0: memos.api.v1.MotionMediaFamily
|
||||
(MotionMediaRole)(0), // 1: memos.api.v1.MotionMediaRole
|
||||
|
|
@ -1245,15 +1525,18 @@ var file_api_v1_attachment_service_proto_goTypes = []any{
|
|||
(*VideoMetadata)(nil), // 7: memos.api.v1.VideoMetadata
|
||||
(*Attachment)(nil), // 8: memos.api.v1.Attachment
|
||||
(*CreateAttachmentRequest)(nil), // 9: memos.api.v1.CreateAttachmentRequest
|
||||
(*ListAttachmentsRequest)(nil), // 10: memos.api.v1.ListAttachmentsRequest
|
||||
(*ListAttachmentsResponse)(nil), // 11: memos.api.v1.ListAttachmentsResponse
|
||||
(*GetAttachmentRequest)(nil), // 12: memos.api.v1.GetAttachmentRequest
|
||||
(*UpdateAttachmentRequest)(nil), // 13: memos.api.v1.UpdateAttachmentRequest
|
||||
(*DeleteAttachmentRequest)(nil), // 14: memos.api.v1.DeleteAttachmentRequest
|
||||
(*BatchDeleteAttachmentsRequest)(nil), // 15: memos.api.v1.BatchDeleteAttachmentsRequest
|
||||
(*timestamppb.Timestamp)(nil), // 16: google.protobuf.Timestamp
|
||||
(*fieldmaskpb.FieldMask)(nil), // 17: google.protobuf.FieldMask
|
||||
(*emptypb.Empty)(nil), // 18: google.protobuf.Empty
|
||||
(*UploadAttachmentRequest)(nil), // 10: memos.api.v1.UploadAttachmentRequest
|
||||
(*UploadAttachmentSpec)(nil), // 11: memos.api.v1.UploadAttachmentSpec
|
||||
(*UploadAttachmentResponse)(nil), // 12: memos.api.v1.UploadAttachmentResponse
|
||||
(*ListAttachmentsRequest)(nil), // 13: memos.api.v1.ListAttachmentsRequest
|
||||
(*ListAttachmentsResponse)(nil), // 14: memos.api.v1.ListAttachmentsResponse
|
||||
(*GetAttachmentRequest)(nil), // 15: memos.api.v1.GetAttachmentRequest
|
||||
(*UpdateAttachmentRequest)(nil), // 16: memos.api.v1.UpdateAttachmentRequest
|
||||
(*DeleteAttachmentRequest)(nil), // 17: memos.api.v1.DeleteAttachmentRequest
|
||||
(*BatchDeleteAttachmentsRequest)(nil), // 18: memos.api.v1.BatchDeleteAttachmentsRequest
|
||||
(*timestamppb.Timestamp)(nil), // 19: google.protobuf.Timestamp
|
||||
(*fieldmaskpb.FieldMask)(nil), // 20: google.protobuf.FieldMask
|
||||
(*emptypb.Empty)(nil), // 21: google.protobuf.Empty
|
||||
}
|
||||
var file_api_v1_attachment_service_proto_depIdxs = []int32{
|
||||
0, // 0: memos.api.v1.MotionMedia.family:type_name -> memos.api.v1.MotionMediaFamily
|
||||
|
|
@ -1262,30 +1545,35 @@ var file_api_v1_attachment_service_proto_depIdxs = []int32{
|
|||
7, // 3: memos.api.v1.MediaMetadata.video:type_name -> memos.api.v1.VideoMetadata
|
||||
5, // 4: memos.api.v1.PhotoMetadata.capture_time:type_name -> memos.api.v1.MediaCaptureTime
|
||||
6, // 5: memos.api.v1.PhotoMetadata.location:type_name -> memos.api.v1.MediaLocation
|
||||
16, // 6: memos.api.v1.Attachment.create_time:type_name -> google.protobuf.Timestamp
|
||||
19, // 6: memos.api.v1.Attachment.create_time:type_name -> google.protobuf.Timestamp
|
||||
2, // 7: memos.api.v1.Attachment.motion_media:type_name -> memos.api.v1.MotionMedia
|
||||
3, // 8: memos.api.v1.Attachment.media_metadata:type_name -> memos.api.v1.MediaMetadata
|
||||
8, // 9: memos.api.v1.CreateAttachmentRequest.attachment:type_name -> memos.api.v1.Attachment
|
||||
8, // 10: memos.api.v1.ListAttachmentsResponse.attachments:type_name -> memos.api.v1.Attachment
|
||||
8, // 11: memos.api.v1.UpdateAttachmentRequest.attachment:type_name -> memos.api.v1.Attachment
|
||||
17, // 12: memos.api.v1.UpdateAttachmentRequest.update_mask:type_name -> google.protobuf.FieldMask
|
||||
9, // 13: memos.api.v1.AttachmentService.CreateAttachment:input_type -> memos.api.v1.CreateAttachmentRequest
|
||||
10, // 14: memos.api.v1.AttachmentService.ListAttachments:input_type -> memos.api.v1.ListAttachmentsRequest
|
||||
12, // 15: memos.api.v1.AttachmentService.GetAttachment:input_type -> memos.api.v1.GetAttachmentRequest
|
||||
13, // 16: memos.api.v1.AttachmentService.UpdateAttachment:input_type -> memos.api.v1.UpdateAttachmentRequest
|
||||
14, // 17: memos.api.v1.AttachmentService.DeleteAttachment:input_type -> memos.api.v1.DeleteAttachmentRequest
|
||||
15, // 18: memos.api.v1.AttachmentService.BatchDeleteAttachments:input_type -> memos.api.v1.BatchDeleteAttachmentsRequest
|
||||
8, // 19: memos.api.v1.AttachmentService.CreateAttachment:output_type -> memos.api.v1.Attachment
|
||||
11, // 20: memos.api.v1.AttachmentService.ListAttachments:output_type -> memos.api.v1.ListAttachmentsResponse
|
||||
8, // 21: memos.api.v1.AttachmentService.GetAttachment:output_type -> memos.api.v1.Attachment
|
||||
8, // 22: memos.api.v1.AttachmentService.UpdateAttachment:output_type -> memos.api.v1.Attachment
|
||||
18, // 23: memos.api.v1.AttachmentService.DeleteAttachment:output_type -> google.protobuf.Empty
|
||||
18, // 24: memos.api.v1.AttachmentService.BatchDeleteAttachments:output_type -> google.protobuf.Empty
|
||||
19, // [19:25] is the sub-list for method output_type
|
||||
13, // [13:19] is the sub-list for method input_type
|
||||
13, // [13:13] is the sub-list for extension type_name
|
||||
13, // [13:13] is the sub-list for extension extendee
|
||||
0, // [0:13] is the sub-list for field type_name
|
||||
11, // 10: memos.api.v1.UploadAttachmentRequest.spec:type_name -> memos.api.v1.UploadAttachmentSpec
|
||||
8, // 11: memos.api.v1.UploadAttachmentSpec.attachment:type_name -> memos.api.v1.Attachment
|
||||
8, // 12: memos.api.v1.UploadAttachmentResponse.attachment:type_name -> memos.api.v1.Attachment
|
||||
8, // 13: memos.api.v1.ListAttachmentsResponse.attachments:type_name -> memos.api.v1.Attachment
|
||||
8, // 14: memos.api.v1.UpdateAttachmentRequest.attachment:type_name -> memos.api.v1.Attachment
|
||||
20, // 15: memos.api.v1.UpdateAttachmentRequest.update_mask:type_name -> google.protobuf.FieldMask
|
||||
9, // 16: memos.api.v1.AttachmentService.CreateAttachment:input_type -> memos.api.v1.CreateAttachmentRequest
|
||||
10, // 17: memos.api.v1.AttachmentService.UploadAttachment:input_type -> memos.api.v1.UploadAttachmentRequest
|
||||
13, // 18: memos.api.v1.AttachmentService.ListAttachments:input_type -> memos.api.v1.ListAttachmentsRequest
|
||||
15, // 19: memos.api.v1.AttachmentService.GetAttachment:input_type -> memos.api.v1.GetAttachmentRequest
|
||||
16, // 20: memos.api.v1.AttachmentService.UpdateAttachment:input_type -> memos.api.v1.UpdateAttachmentRequest
|
||||
17, // 21: memos.api.v1.AttachmentService.DeleteAttachment:input_type -> memos.api.v1.DeleteAttachmentRequest
|
||||
18, // 22: memos.api.v1.AttachmentService.BatchDeleteAttachments:input_type -> memos.api.v1.BatchDeleteAttachmentsRequest
|
||||
8, // 23: memos.api.v1.AttachmentService.CreateAttachment:output_type -> memos.api.v1.Attachment
|
||||
12, // 24: memos.api.v1.AttachmentService.UploadAttachment:output_type -> memos.api.v1.UploadAttachmentResponse
|
||||
14, // 25: memos.api.v1.AttachmentService.ListAttachments:output_type -> memos.api.v1.ListAttachmentsResponse
|
||||
8, // 26: memos.api.v1.AttachmentService.GetAttachment:output_type -> memos.api.v1.Attachment
|
||||
8, // 27: memos.api.v1.AttachmentService.UpdateAttachment:output_type -> memos.api.v1.Attachment
|
||||
21, // 28: memos.api.v1.AttachmentService.DeleteAttachment:output_type -> google.protobuf.Empty
|
||||
21, // 29: memos.api.v1.AttachmentService.BatchDeleteAttachments:output_type -> google.protobuf.Empty
|
||||
23, // [23:30] is the sub-list for method output_type
|
||||
16, // [16:23] is the sub-list for method input_type
|
||||
16, // [16:16] is the sub-list for extension type_name
|
||||
16, // [16:16] is the sub-list for extension extendee
|
||||
0, // [0:16] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_api_v1_attachment_service_proto_init() }
|
||||
|
|
@ -1302,13 +1590,17 @@ func file_api_v1_attachment_service_proto_init() {
|
|||
file_api_v1_attachment_service_proto_msgTypes[4].OneofWrappers = []any{}
|
||||
file_api_v1_attachment_service_proto_msgTypes[5].OneofWrappers = []any{}
|
||||
file_api_v1_attachment_service_proto_msgTypes[6].OneofWrappers = []any{}
|
||||
file_api_v1_attachment_service_proto_msgTypes[8].OneofWrappers = []any{
|
||||
(*UploadAttachmentRequest_Spec)(nil),
|
||||
(*UploadAttachmentRequest_UploadId)(nil),
|
||||
}
|
||||
type x struct{}
|
||||
out := protoimpl.TypeBuilder{
|
||||
File: protoimpl.DescBuilder{
|
||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: unsafe.Slice(unsafe.StringData(file_api_v1_attachment_service_proto_rawDesc), len(file_api_v1_attachment_service_proto_rawDesc)),
|
||||
NumEnums: 2,
|
||||
NumMessages: 14,
|
||||
NumMessages: 17,
|
||||
NumExtensions: 0,
|
||||
NumServices: 1,
|
||||
},
|
||||
|
|
|
|||
|
|
@ -76,6 +76,33 @@ func local_request_AttachmentService_CreateAttachment_0(ctx context.Context, mar
|
|||
return msg, metadata, err
|
||||
}
|
||||
|
||||
func request_AttachmentService_UploadAttachment_0(ctx context.Context, marshaler runtime.Marshaler, client AttachmentServiceClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) {
|
||||
var (
|
||||
protoReq UploadAttachmentRequest
|
||||
metadata runtime.ServerMetadata
|
||||
)
|
||||
if err := marshaler.NewDecoder(req.Body).Decode(&protoReq); err != nil && !errors.Is(err, io.EOF) {
|
||||
return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err)
|
||||
}
|
||||
if req.Body != nil {
|
||||
_, _ = io.Copy(io.Discard, req.Body)
|
||||
}
|
||||
msg, err := client.UploadAttachment(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD))
|
||||
return msg, metadata, err
|
||||
}
|
||||
|
||||
func local_request_AttachmentService_UploadAttachment_0(ctx context.Context, marshaler runtime.Marshaler, server AttachmentServiceServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) {
|
||||
var (
|
||||
protoReq UploadAttachmentRequest
|
||||
metadata runtime.ServerMetadata
|
||||
)
|
||||
if err := marshaler.NewDecoder(req.Body).Decode(&protoReq); err != nil && !errors.Is(err, io.EOF) {
|
||||
return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err)
|
||||
}
|
||||
msg, err := server.UploadAttachment(ctx, &protoReq)
|
||||
return msg, metadata, err
|
||||
}
|
||||
|
||||
var filter_AttachmentService_ListAttachments_0 = &utilities.DoubleArray{Encoding: map[string]int{}, Base: []int(nil), Check: []int(nil)}
|
||||
|
||||
func request_AttachmentService_ListAttachments_0(ctx context.Context, marshaler runtime.Marshaler, client AttachmentServiceClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) {
|
||||
|
|
@ -323,6 +350,26 @@ func RegisterAttachmentServiceHandlerServer(ctx context.Context, mux *runtime.Se
|
|||
}
|
||||
forward_AttachmentService_CreateAttachment_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...)
|
||||
})
|
||||
mux.Handle(http.MethodPost, pattern_AttachmentService_UploadAttachment_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
|
||||
ctx, cancel := context.WithCancel(req.Context())
|
||||
defer cancel()
|
||||
var stream runtime.ServerTransportStream
|
||||
ctx = grpc.NewContextWithServerTransportStream(ctx, &stream)
|
||||
inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req)
|
||||
annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/memos.api.v1.AttachmentService/UploadAttachment", runtime.WithHTTPPathPattern("/api/v1/attachments:upload"))
|
||||
if err != nil {
|
||||
runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
|
||||
return
|
||||
}
|
||||
resp, md, err := local_request_AttachmentService_UploadAttachment_0(annotatedContext, inboundMarshaler, server, req, pathParams)
|
||||
md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer())
|
||||
annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md)
|
||||
if err != nil {
|
||||
runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err)
|
||||
return
|
||||
}
|
||||
forward_AttachmentService_UploadAttachment_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...)
|
||||
})
|
||||
mux.Handle(http.MethodGet, pattern_AttachmentService_ListAttachments_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
|
||||
ctx, cancel := context.WithCancel(req.Context())
|
||||
defer cancel()
|
||||
|
|
@ -480,6 +527,23 @@ func RegisterAttachmentServiceHandlerClient(ctx context.Context, mux *runtime.Se
|
|||
}
|
||||
forward_AttachmentService_CreateAttachment_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...)
|
||||
})
|
||||
mux.Handle(http.MethodPost, pattern_AttachmentService_UploadAttachment_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
|
||||
ctx, cancel := context.WithCancel(req.Context())
|
||||
defer cancel()
|
||||
inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req)
|
||||
annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/memos.api.v1.AttachmentService/UploadAttachment", runtime.WithHTTPPathPattern("/api/v1/attachments:upload"))
|
||||
if err != nil {
|
||||
runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err)
|
||||
return
|
||||
}
|
||||
resp, md, err := request_AttachmentService_UploadAttachment_0(annotatedContext, inboundMarshaler, client, req, pathParams)
|
||||
annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md)
|
||||
if err != nil {
|
||||
runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err)
|
||||
return
|
||||
}
|
||||
forward_AttachmentService_UploadAttachment_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...)
|
||||
})
|
||||
mux.Handle(http.MethodGet, pattern_AttachmentService_ListAttachments_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
|
||||
ctx, cancel := context.WithCancel(req.Context())
|
||||
defer cancel()
|
||||
|
|
@ -570,6 +634,7 @@ func RegisterAttachmentServiceHandlerClient(ctx context.Context, mux *runtime.Se
|
|||
|
||||
var (
|
||||
pattern_AttachmentService_CreateAttachment_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"api", "v1", "attachments"}, ""))
|
||||
pattern_AttachmentService_UploadAttachment_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"api", "v1", "attachments"}, "upload"))
|
||||
pattern_AttachmentService_ListAttachments_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"api", "v1", "attachments"}, ""))
|
||||
pattern_AttachmentService_GetAttachment_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 1, 0, 4, 2, 5, 3}, []string{"api", "v1", "attachments", "name"}, ""))
|
||||
pattern_AttachmentService_UpdateAttachment_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 1, 0, 4, 2, 5, 3}, []string{"api", "v1", "attachments", "attachment.name"}, ""))
|
||||
|
|
@ -579,6 +644,7 @@ var (
|
|||
|
||||
var (
|
||||
forward_AttachmentService_CreateAttachment_0 = runtime.ForwardResponseMessage
|
||||
forward_AttachmentService_UploadAttachment_0 = runtime.ForwardResponseMessage
|
||||
forward_AttachmentService_ListAttachments_0 = runtime.ForwardResponseMessage
|
||||
forward_AttachmentService_GetAttachment_0 = runtime.ForwardResponseMessage
|
||||
forward_AttachmentService_UpdateAttachment_0 = runtime.ForwardResponseMessage
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ const _ = grpc.SupportPackageIsVersion9
|
|||
|
||||
const (
|
||||
AttachmentService_CreateAttachment_FullMethodName = "/memos.api.v1.AttachmentService/CreateAttachment"
|
||||
AttachmentService_UploadAttachment_FullMethodName = "/memos.api.v1.AttachmentService/UploadAttachment"
|
||||
AttachmentService_ListAttachments_FullMethodName = "/memos.api.v1.AttachmentService/ListAttachments"
|
||||
AttachmentService_GetAttachment_FullMethodName = "/memos.api.v1.AttachmentService/GetAttachment"
|
||||
AttachmentService_UpdateAttachment_FullMethodName = "/memos.api.v1.AttachmentService/UpdateAttachment"
|
||||
|
|
@ -34,6 +35,11 @@ const (
|
|||
type AttachmentServiceClient interface {
|
||||
// CreateAttachment creates a new attachment.
|
||||
CreateAttachment(ctx context.Context, in *CreateAttachmentRequest, opts ...grpc.CallOption) (*Attachment, error)
|
||||
// UploadAttachment uploads a file in bounded chunks. The first call carries
|
||||
// the spec and returns an upload_id; later calls carry that upload_id.
|
||||
// Uploads are bound to the authenticated user, expire after 30 minutes of
|
||||
// inactivity, and do not survive a server restart.
|
||||
UploadAttachment(ctx context.Context, in *UploadAttachmentRequest, opts ...grpc.CallOption) (*UploadAttachmentResponse, error)
|
||||
// ListAttachments lists all attachments.
|
||||
ListAttachments(ctx context.Context, in *ListAttachmentsRequest, opts ...grpc.CallOption) (*ListAttachmentsResponse, error)
|
||||
// GetAttachment returns an attachment by name.
|
||||
|
|
@ -64,6 +70,16 @@ func (c *attachmentServiceClient) CreateAttachment(ctx context.Context, in *Crea
|
|||
return out, nil
|
||||
}
|
||||
|
||||
func (c *attachmentServiceClient) UploadAttachment(ctx context.Context, in *UploadAttachmentRequest, opts ...grpc.CallOption) (*UploadAttachmentResponse, error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
out := new(UploadAttachmentResponse)
|
||||
err := c.cc.Invoke(ctx, AttachmentService_UploadAttachment_FullMethodName, in, out, cOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *attachmentServiceClient) ListAttachments(ctx context.Context, in *ListAttachmentsRequest, opts ...grpc.CallOption) (*ListAttachmentsResponse, error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
out := new(ListAttachmentsResponse)
|
||||
|
|
@ -120,6 +136,11 @@ func (c *attachmentServiceClient) BatchDeleteAttachments(ctx context.Context, in
|
|||
type AttachmentServiceServer interface {
|
||||
// CreateAttachment creates a new attachment.
|
||||
CreateAttachment(context.Context, *CreateAttachmentRequest) (*Attachment, error)
|
||||
// UploadAttachment uploads a file in bounded chunks. The first call carries
|
||||
// the spec and returns an upload_id; later calls carry that upload_id.
|
||||
// Uploads are bound to the authenticated user, expire after 30 minutes of
|
||||
// inactivity, and do not survive a server restart.
|
||||
UploadAttachment(context.Context, *UploadAttachmentRequest) (*UploadAttachmentResponse, error)
|
||||
// ListAttachments lists all attachments.
|
||||
ListAttachments(context.Context, *ListAttachmentsRequest) (*ListAttachmentsResponse, error)
|
||||
// GetAttachment returns an attachment by name.
|
||||
|
|
@ -143,6 +164,9 @@ type UnimplementedAttachmentServiceServer struct{}
|
|||
func (UnimplementedAttachmentServiceServer) CreateAttachment(context.Context, *CreateAttachmentRequest) (*Attachment, error) {
|
||||
return nil, status.Error(codes.Unimplemented, "method CreateAttachment not implemented")
|
||||
}
|
||||
func (UnimplementedAttachmentServiceServer) UploadAttachment(context.Context, *UploadAttachmentRequest) (*UploadAttachmentResponse, error) {
|
||||
return nil, status.Error(codes.Unimplemented, "method UploadAttachment not implemented")
|
||||
}
|
||||
func (UnimplementedAttachmentServiceServer) ListAttachments(context.Context, *ListAttachmentsRequest) (*ListAttachmentsResponse, error) {
|
||||
return nil, status.Error(codes.Unimplemented, "method ListAttachments not implemented")
|
||||
}
|
||||
|
|
@ -197,6 +221,24 @@ func _AttachmentService_CreateAttachment_Handler(srv interface{}, ctx context.Co
|
|||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _AttachmentService_UploadAttachment_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(UploadAttachmentRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(AttachmentServiceServer).UploadAttachment(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: AttachmentService_UploadAttachment_FullMethodName,
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(AttachmentServiceServer).UploadAttachment(ctx, req.(*UploadAttachmentRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _AttachmentService_ListAttachments_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(ListAttachmentsRequest)
|
||||
if err := dec(in); err != nil {
|
||||
|
|
@ -298,6 +340,10 @@ var AttachmentService_ServiceDesc = grpc.ServiceDesc{
|
|||
MethodName: "CreateAttachment",
|
||||
Handler: _AttachmentService_CreateAttachment_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "UploadAttachment",
|
||||
Handler: _AttachmentService_UploadAttachment_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "ListAttachments",
|
||||
Handler: _AttachmentService_ListAttachments_Handler,
|
||||
|
|
|
|||
|
|
@ -228,6 +228,35 @@ paths:
|
|||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Status'
|
||||
/api/v1/attachments:upload:
|
||||
post:
|
||||
tags:
|
||||
- AttachmentService
|
||||
description: |-
|
||||
UploadAttachment uploads a file in bounded chunks. The first call carries
|
||||
the spec and returns an upload_id; later calls carry that upload_id.
|
||||
Uploads are bound to the authenticated user, expire after 30 minutes of
|
||||
inactivity, and do not survive a server restart.
|
||||
operationId: AttachmentService_UploadAttachment
|
||||
requestBody:
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/UploadAttachmentRequest'
|
||||
required: true
|
||||
responses:
|
||||
"200":
|
||||
description: OK
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/UploadAttachmentResponse'
|
||||
default:
|
||||
description: Default error response
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Status'
|
||||
/api/v1/auth/me:
|
||||
get:
|
||||
tags:
|
||||
|
|
@ -4573,6 +4602,76 @@ components:
|
|||
contentType:
|
||||
type: string
|
||||
description: Optional. The MIME type of the input audio.
|
||||
UploadAttachmentRequest:
|
||||
required:
|
||||
- writeOffset
|
||||
type: object
|
||||
properties:
|
||||
spec:
|
||||
allOf:
|
||||
- $ref: '#/components/schemas/UploadAttachmentSpec'
|
||||
description: Starts a new upload. The same call may also carry data and finish_write.
|
||||
uploadId:
|
||||
type: string
|
||||
description: Continues the upload identified by a previous response.
|
||||
writeOffset:
|
||||
type: string
|
||||
description: |-
|
||||
Required. Zero-based byte offset at which data is written. Must equal the
|
||||
committed size, except when retrying the most recently accepted chunk
|
||||
with identical bytes and offset, which is accepted without writing again.
|
||||
data:
|
||||
type: string
|
||||
description: |-
|
||||
Optional. File bytes, at most max_chunk_size long. With no data and
|
||||
finish_write false, the call reports progress without writing, and
|
||||
write_offset is ignored.
|
||||
format: bytes
|
||||
finishWrite:
|
||||
type: boolean
|
||||
description: |-
|
||||
Optional. Finalize the upload after writing data. The committed size must
|
||||
then equal total_size. Any later call for the same upload_id returns the
|
||||
created attachment.
|
||||
UploadAttachmentResponse:
|
||||
type: object
|
||||
properties:
|
||||
uploadId:
|
||||
type: string
|
||||
description: Opaque ID for subsequent calls. This is not a resource name.
|
||||
committedSize:
|
||||
type: string
|
||||
description: Number of file bytes committed so far.
|
||||
attachment:
|
||||
allOf:
|
||||
- $ref: '#/components/schemas/Attachment'
|
||||
description: Set once the upload has been finalized.
|
||||
maxChunkSize:
|
||||
type: integer
|
||||
description: Maximum number of data bytes accepted in one call.
|
||||
format: int32
|
||||
UploadAttachmentSpec:
|
||||
required:
|
||||
- attachment
|
||||
type: object
|
||||
properties:
|
||||
attachment:
|
||||
allOf:
|
||||
- $ref: '#/components/schemas/Attachment'
|
||||
description: |-
|
||||
Required. Metadata for the attachment to create. content must be empty;
|
||||
file bytes are sent in data.
|
||||
attachmentId:
|
||||
type: string
|
||||
description: |-
|
||||
Optional. The attachment ID to use for this attachment.
|
||||
If empty, a unique ID will be generated.
|
||||
Format: ^[a-zA-Z0-9]([a-zA-Z0-9-]{0,34}[a-zA-Z0-9])?$
|
||||
totalSize:
|
||||
type: string
|
||||
description: |-
|
||||
Optional. Total size of the file in bytes before media processing.
|
||||
Zero represents an empty file.
|
||||
UpsertMemoReactionRequest:
|
||||
required:
|
||||
- name
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ func TestPublicMethodsArePublic(t *testing.T) {
|
|||
// TestProtectedMethodsRequireAuth verifies that non-public methods are recognized as protected.
|
||||
func TestProtectedMethodsRequireAuth(t *testing.T) {
|
||||
protectedMethods := []string{
|
||||
"/memos.api.v1.AttachmentService/UploadAttachment",
|
||||
// Auth Service - logout and get current user require auth
|
||||
"/memos.api.v1.AuthService/SignOut",
|
||||
"/memos.api.v1.AuthService/GetCurrentUser",
|
||||
|
|
|
|||
|
|
@ -104,7 +104,7 @@ func TestStripImageExif(t *testing.T) {
|
|||
t.Run("strip JPEG metadata", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
strippedData, err := stripImageExif(originalData, "image/jpeg")
|
||||
strippedData, err := stripImageExif(bytes.NewReader(originalData), "image/jpeg")
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, strippedData)
|
||||
|
||||
|
|
@ -118,7 +118,7 @@ func TestStripImageExif(t *testing.T) {
|
|||
t.Run("strip JPG metadata (alternate extension)", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
strippedData, err := stripImageExif(originalData, "image/jpg")
|
||||
strippedData, err := stripImageExif(bytes.NewReader(originalData), "image/jpg")
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, strippedData)
|
||||
|
||||
|
|
@ -136,7 +136,7 @@ func TestStripImageExif(t *testing.T) {
|
|||
err := imaging.Encode(&pngBuf, img, imaging.PNG)
|
||||
require.NoError(t, err)
|
||||
|
||||
strippedData, err := stripImageExif(pngBuf.Bytes(), "image/png")
|
||||
strippedData, err := stripImageExif(bytes.NewReader(pngBuf.Bytes()), "image/png")
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, strippedData)
|
||||
|
||||
|
|
@ -151,7 +151,7 @@ func TestStripImageExif(t *testing.T) {
|
|||
t.Parallel()
|
||||
|
||||
// WebP format will be converted to JPEG
|
||||
strippedData, err := stripImageExif(originalData, "image/webp")
|
||||
strippedData, err := stripImageExif(bytes.NewReader(originalData), "image/webp")
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, strippedData)
|
||||
|
||||
|
|
@ -164,7 +164,7 @@ func TestStripImageExif(t *testing.T) {
|
|||
t.Run("handle HEIC format by converting to JPEG", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
strippedData, err := stripImageExif(originalData, "image/heic")
|
||||
strippedData, err := stripImageExif(bytes.NewReader(originalData), "image/heic")
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, strippedData)
|
||||
|
||||
|
|
@ -178,7 +178,7 @@ func TestStripImageExif(t *testing.T) {
|
|||
t.Parallel()
|
||||
|
||||
invalidData := []byte("not an image")
|
||||
_, err := stripImageExif(invalidData, "image/jpeg")
|
||||
_, err := stripImageExif(bytes.NewReader(invalidData), "image/jpeg")
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "failed to decode image")
|
||||
})
|
||||
|
|
@ -187,7 +187,7 @@ func TestStripImageExif(t *testing.T) {
|
|||
t.Parallel()
|
||||
|
||||
emptyData := []byte{}
|
||||
_, err := stripImageExif(emptyData, "image/jpeg")
|
||||
_, err := stripImageExif(bytes.NewReader(emptyData), "image/jpeg")
|
||||
assert.Error(t, err)
|
||||
})
|
||||
}
|
||||
|
|
@ -195,7 +195,7 @@ func TestStripImageExif(t *testing.T) {
|
|||
func TestValidateImagePixelCountRejectsOversizedDimensions(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
err := validateImagePixelCount(testPNGHeaderWithDimensions(100_000, 100_000))
|
||||
err := validateImageReaderPixelCount(bytes.NewReader(testPNGHeaderWithDimensions(100_000, 100_000)))
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "image dimensions exceed maximum")
|
||||
}
|
||||
|
|
@ -203,7 +203,7 @@ func TestValidateImagePixelCountRejectsOversizedDimensions(t *testing.T) {
|
|||
func TestStripImageExifRejectsOversizedDimensionsBeforeDecode(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := stripImageExif(testPNGHeaderWithDimensions(100_000, 100_000), "image/png")
|
||||
_, err := stripImageExif(bytes.NewReader(testPNGHeaderWithDimensions(100_000, 100_000)), "image/png")
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "image dimensions exceed maximum")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
package v1
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"math"
|
||||
"mime"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
|
|
@ -15,16 +17,16 @@ import (
|
|||
"google.golang.org/grpc/status"
|
||||
"google.golang.org/protobuf/types/known/emptypb"
|
||||
|
||||
"github.com/usememos/memos/internal/motionphoto"
|
||||
v1pb "github.com/usememos/memos/proto/gen/api/v1"
|
||||
storepb "github.com/usememos/memos/proto/gen/store"
|
||||
"github.com/usememos/memos/store"
|
||||
)
|
||||
|
||||
const (
|
||||
// The upload memory buffer is 32 MiB.
|
||||
// It should be kept low, so RAM usage doesn't get out of control.
|
||||
// This is unrelated to maximum upload size limit, which is now set through system setting.
|
||||
MaxUploadBufferSizeBytes = 32 << 20
|
||||
MebiByte = 1024 * 1024
|
||||
// DefaultUploadSizeLimitBytes applies when no upload size limit is configured.
|
||||
DefaultUploadSizeLimitBytes = 32 << 20
|
||||
MebiByte = 1024 * 1024
|
||||
|
||||
// defaultJPEGQuality is the JPEG quality used when re-encoding images for EXIF stripping.
|
||||
// Quality 95 maintains visual quality while ensuring metadata is removed.
|
||||
|
|
@ -71,7 +73,7 @@ func detectAttachmentMimeType(filename string, content []byte) string {
|
|||
return http.DetectContentType(content)
|
||||
}
|
||||
|
||||
func (s *APIV1Service) CreateAttachment(ctx context.Context, request *v1pb.CreateAttachmentRequest) (*v1pb.Attachment, error) {
|
||||
func (s *APIV1Service) prepareAttachment(ctx context.Context, request *v1pb.CreateAttachmentRequest) (*store.Attachment, error) {
|
||||
user, err := s.fetchCurrentUser(ctx)
|
||||
if err != nil {
|
||||
return nil, status.Errorf(codes.Internal, "failed to get current user: %v", err)
|
||||
|
|
@ -135,21 +137,6 @@ func (s *APIV1Service) CreateAttachment(ctx context.Context, request *v1pb.Creat
|
|||
create.Payload.MediaMetadata = inputMediaMetadata
|
||||
}
|
||||
|
||||
instanceStorageSetting, err := s.Store.GetInstanceStorageSetting(ctx)
|
||||
if err != nil {
|
||||
return nil, status.Errorf(codes.Internal, "failed to get instance storage setting: %v", err)
|
||||
}
|
||||
size := binary.Size(request.Attachment.Content)
|
||||
uploadSizeLimit := int(instanceStorageSetting.UploadSizeLimitMb) * MebiByte
|
||||
if uploadSizeLimit == 0 {
|
||||
uploadSizeLimit = MaxUploadBufferSizeBytes
|
||||
}
|
||||
if size > uploadSizeLimit {
|
||||
return nil, status.Errorf(codes.InvalidArgument, "file size exceeds the limit")
|
||||
}
|
||||
create.Size = int64(size)
|
||||
create.Blob = request.Attachment.Content
|
||||
|
||||
if request.Attachment.Memo != nil {
|
||||
memoUID, err := ExtractMemoUIDFromName(*request.Attachment.Memo)
|
||||
if err != nil {
|
||||
|
|
@ -172,21 +159,77 @@ func (s *APIV1Service) CreateAttachment(ctx context.Context, request *v1pb.Creat
|
|||
create.Policy = memoWritePolicy(user.ID, false)
|
||||
}
|
||||
|
||||
if create.Payload == nil || create.Payload.MotionMedia == nil {
|
||||
if detectedMotion := detectAndroidMotionMedia(create.Blob, create.Type, attachmentUID); detectedMotion != nil {
|
||||
return create, nil
|
||||
}
|
||||
|
||||
func (s *APIV1Service) CreateAttachment(ctx context.Context, request *v1pb.CreateAttachmentRequest) (*v1pb.Attachment, error) {
|
||||
create, err := s.prepareAttachment(ctx, request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
instanceStorageSetting, err := s.Store.GetInstanceStorageSetting(ctx)
|
||||
if err != nil {
|
||||
return nil, status.Errorf(codes.Internal, "failed to get instance storage setting: %v", err)
|
||||
}
|
||||
content := request.Attachment.Content
|
||||
if err := checkUploadSize(instanceStorageSetting, int64(len(content))); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
create.Size = int64(len(content))
|
||||
return s.processAndSaveAttachment(ctx, create, instanceStorageSetting, bytes.NewReader(content))
|
||||
}
|
||||
|
||||
func attachmentUploadLimit(setting *storepb.InstanceStorageSetting) int64 {
|
||||
if setting.UploadSizeLimitMb <= 0 {
|
||||
return DefaultUploadSizeLimitBytes
|
||||
}
|
||||
return min(setting.UploadSizeLimitMb, math.MaxInt64/MebiByte) * MebiByte
|
||||
}
|
||||
|
||||
func checkUploadSize(setting *storepb.InstanceStorageSetting, size int64) error {
|
||||
if size > attachmentUploadLimit(setting) {
|
||||
return status.Errorf(codes.ResourceExhausted, "file size exceeds the limit")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// attachmentSource is the file content handed to the processing pipeline: a
|
||||
// bytes.Reader for the one-shot RPC, an *os.File for a chunked upload.
|
||||
type attachmentSource interface {
|
||||
io.ReadSeeker
|
||||
io.ReaderAt
|
||||
}
|
||||
|
||||
// processAndSaveAttachment detects motion photos, strips EXIF metadata, stores
|
||||
// the content, and creates the database row. create.Size must hold the source
|
||||
// length on entry; it is updated when stripping re-encodes the image.
|
||||
func (s *APIV1Service) processAndSaveAttachment(ctx context.Context, create *store.Attachment, instanceStorageSetting *storepb.InstanceStorageSetting, source attachmentSource) (*v1pb.Attachment, error) {
|
||||
if create.Payload.GetMotionMedia() == nil && (create.Type == "image/jpeg" || create.Type == "image/jpg") {
|
||||
detected, err := motionphoto.DetectJPEGReader(source, create.Size)
|
||||
if err != nil {
|
||||
return nil, status.Errorf(codes.Internal, "failed to inspect motion photo: %v", err)
|
||||
}
|
||||
if detected != nil {
|
||||
create.Payload = ensureAttachmentPayload(create.Payload)
|
||||
create.Payload.MotionMedia = detectedMotion
|
||||
create.Payload.MotionMedia = &storepb.MotionMedia{
|
||||
Family: storepb.MotionMediaFamily_ANDROID_MOTION_PHOTO,
|
||||
Role: storepb.MotionMediaRole_CONTAINER,
|
||||
GroupId: create.UID,
|
||||
PresentationTimestampUs: detected.PresentationTimestampUs,
|
||||
HasEmbeddedVideo: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Strip EXIF metadata from images for privacy protection.
|
||||
// This removes sensitive information like GPS location, device details, etc.
|
||||
content := io.ReadSeeker(source)
|
||||
// Strip EXIF metadata from images for privacy protection. Motion photo
|
||||
// containers are kept intact because re-encoding would drop the video.
|
||||
if shouldStripExif(create.Type) && !isAndroidMotionContainer(create.Payload.GetMotionMedia()) {
|
||||
release, err := s.acquireImageProcessingSlot(ctx)
|
||||
if err != nil {
|
||||
return nil, status.Errorf(codes.ResourceExhausted, "too many image processing requests")
|
||||
}
|
||||
strippedBlob, stripErr := stripImageExif(create.Blob, create.Type)
|
||||
stripped, stripErr := stripImageExif(source, create.Type)
|
||||
release()
|
||||
if stripErr != nil {
|
||||
// Log warning but continue with original image to ensure uploads don't fail.
|
||||
|
|
@ -195,15 +238,21 @@ func (s *APIV1Service) CreateAttachment(ctx context.Context, request *v1pb.Creat
|
|||
slog.String("filename", create.Filename),
|
||||
slog.String("error", stripErr.Error()))
|
||||
} else {
|
||||
create.Blob = strippedBlob
|
||||
create.Size = int64(len(strippedBlob))
|
||||
content = bytes.NewReader(stripped)
|
||||
create.Size = int64(len(stripped))
|
||||
}
|
||||
}
|
||||
|
||||
if err := saveAttachmentBlobWithInstanceStorageSetting(ctx, s.Profile, s.Store, create, instanceStorageSetting); err != nil {
|
||||
if _, err := content.Seek(0, io.SeekStart); err != nil {
|
||||
return nil, status.Errorf(codes.Internal, "failed to rewind attachment content: %v", err)
|
||||
}
|
||||
if err := saveAttachmentContent(ctx, s.Profile, s.Store, create, instanceStorageSetting, content); err != nil {
|
||||
return nil, status.Errorf(codes.Internal, "failed to save attachment blob: %v", err)
|
||||
}
|
||||
return s.persistAttachment(ctx, create, instanceStorageSetting)
|
||||
}
|
||||
|
||||
func (s *APIV1Service) persistAttachment(ctx context.Context, create *store.Attachment, instanceStorageSetting *storepb.InstanceStorageSetting) (*v1pb.Attachment, error) {
|
||||
attachment, err := s.Store.CreateAttachment(ctx, create)
|
||||
if err != nil {
|
||||
createErr := mapMemoWriteError(err, "failed to create attachment")
|
||||
|
|
|
|||
|
|
@ -4,13 +4,13 @@ import (
|
|||
"bytes"
|
||||
"context"
|
||||
"image"
|
||||
"io"
|
||||
|
||||
"github.com/disintegration/imaging"
|
||||
"github.com/pkg/errors"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
|
||||
"github.com/usememos/memos/internal/motionphoto"
|
||||
v1pb "github.com/usememos/memos/proto/gen/api/v1"
|
||||
storepb "github.com/usememos/memos/proto/gen/store"
|
||||
)
|
||||
|
|
@ -38,25 +38,6 @@ func validateClientMotionMedia(motion *v1pb.MotionMedia, attachmentUID string) (
|
|||
return storeMotion, nil
|
||||
}
|
||||
|
||||
func detectAndroidMotionMedia(blob []byte, mimeType, attachmentUID string) *storepb.MotionMedia {
|
||||
if mimeType != "image/jpeg" && mimeType != "image/jpg" {
|
||||
return nil
|
||||
}
|
||||
|
||||
detection := motionphoto.DetectJPEG(blob)
|
||||
if detection == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &storepb.MotionMedia{
|
||||
Family: storepb.MotionMediaFamily_ANDROID_MOTION_PHOTO,
|
||||
Role: storepb.MotionMediaRole_CONTAINER,
|
||||
GroupId: attachmentUID,
|
||||
PresentationTimestampUs: detection.PresentationTimestampUs,
|
||||
HasEmbeddedVideo: true,
|
||||
}
|
||||
}
|
||||
|
||||
// shouldStripExif checks if the MIME type is an image format that may contain EXIF metadata.
|
||||
// Returns true for formats like JPEG, TIFF, WebP, HEIC, and HEIF which commonly contain
|
||||
// privacy-sensitive metadata such as GPS coordinates, camera settings, and device information.
|
||||
|
|
@ -76,8 +57,8 @@ func (s *APIV1Service) acquireImageProcessingSlot(ctx context.Context) (func(),
|
|||
}, nil
|
||||
}
|
||||
|
||||
func validateImagePixelCount(imageData []byte) error {
|
||||
config, _, err := image.DecodeConfig(bytes.NewReader(imageData))
|
||||
func validateImageReaderPixelCount(reader io.Reader) error {
|
||||
config, _, err := image.DecodeConfig(reader)
|
||||
if err != nil {
|
||||
// Some formats supported by imaging do not expose dimensions through
|
||||
// the standard image registry. Let the full decoder handle those.
|
||||
|
|
@ -97,7 +78,8 @@ func validateImagePixelCount(imageData []byte) error {
|
|||
//
|
||||
// The function preserves the correct image orientation by applying EXIF orientation tags
|
||||
// during decoding before stripping all metadata. Images are re-encoded with high quality
|
||||
// to minimize visual degradation.
|
||||
// to minimize visual degradation. The re-encoded output is returned in memory; its size
|
||||
// is bounded by maxImagePixels, which the decoder already has to hold.
|
||||
//
|
||||
// Supported formats:
|
||||
// - JPEG/JPG: Re-encoded as JPEG with quality 95
|
||||
|
|
@ -105,34 +87,28 @@ func validateImagePixelCount(imageData []byte) error {
|
|||
// - TIFF/WebP/HEIC/HEIF: Re-encoded as JPEG with quality 95
|
||||
//
|
||||
// Returns the cleaned image data without any EXIF metadata, or an error if processing fails.
|
||||
func stripImageExif(imageData []byte, mimeType string) ([]byte, error) {
|
||||
if err := validateImagePixelCount(imageData); err != nil {
|
||||
func stripImageExif(source io.ReadSeeker, mimeType string) ([]byte, error) {
|
||||
if _, err := source.Seek(0, io.SeekStart); err != nil {
|
||||
return nil, errors.Wrap(err, "failed to rewind image")
|
||||
}
|
||||
if err := validateImageReaderPixelCount(source); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Decode image with automatic EXIF orientation correction.
|
||||
// This ensures the image displays correctly after metadata removal.
|
||||
img, err := imaging.Decode(bytes.NewReader(imageData), imaging.AutoOrientation(true))
|
||||
if _, err := source.Seek(0, io.SeekStart); err != nil {
|
||||
return nil, errors.Wrap(err, "failed to rewind image")
|
||||
}
|
||||
img, err := imaging.Decode(source, imaging.AutoOrientation(true))
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to decode image")
|
||||
}
|
||||
|
||||
// Re-encode the image without EXIF metadata.
|
||||
var buf bytes.Buffer
|
||||
var encodeErr error
|
||||
|
||||
if mimeType == "image/png" {
|
||||
// Preserve PNG format for lossless encoding
|
||||
encodeErr = imaging.Encode(&buf, img, imaging.PNG)
|
||||
err = imaging.Encode(&buf, img, imaging.PNG)
|
||||
} else {
|
||||
// For JPEG, TIFF, WebP, HEIC, HEIF - re-encode as JPEG.
|
||||
// This ensures EXIF is stripped and provides good compression.
|
||||
encodeErr = imaging.Encode(&buf, img, imaging.JPEG, imaging.JPEGQuality(defaultJPEGQuality))
|
||||
err = imaging.Encode(&buf, img, imaging.JPEG, imaging.JPEGQuality(defaultJPEGQuality))
|
||||
}
|
||||
|
||||
if encodeErr != nil {
|
||||
return nil, errors.Wrap(encodeErr, "failed to encode image")
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to encode image")
|
||||
}
|
||||
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
package v1
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
|
|
@ -46,21 +45,29 @@ func convertAttachmentFromStore(attachment *store.Attachment) *v1pb.Attachment {
|
|||
return attachmentMessage
|
||||
}
|
||||
|
||||
// SaveAttachmentBlob saves the blob of attachment based on the storage config.
|
||||
func SaveAttachmentBlob(ctx context.Context, profile *profile.Profile, stores *store.Store, create *store.Attachment) error {
|
||||
instanceStorageSetting, err := stores.GetInstanceStorageSetting(ctx)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "Failed to find instance storage setting")
|
||||
}
|
||||
return saveAttachmentBlobWithInstanceStorageSetting(ctx, profile, stores, create, instanceStorageSetting)
|
||||
// attachmentContextReader stops a long local copy once the request context is
|
||||
// canceled; io.Copy has no context of its own, unlike the S3 client.
|
||||
type attachmentContextReader struct {
|
||||
ctx context.Context
|
||||
reader io.Reader
|
||||
}
|
||||
|
||||
func saveAttachmentBlobWithInstanceStorageSetting(
|
||||
func (r *attachmentContextReader) Read(p []byte) (int, error) {
|
||||
if err := r.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return r.reader.Read(p)
|
||||
}
|
||||
|
||||
// saveAttachmentContent writes content to the default storage and records
|
||||
// where it went on create. Database storage keeps the bytes on create.Blob.
|
||||
func saveAttachmentContent(
|
||||
ctx context.Context,
|
||||
profile *profile.Profile,
|
||||
stores *store.Store,
|
||||
create *store.Attachment,
|
||||
instanceStorageSetting *storepb.InstanceStorageSetting,
|
||||
content io.Reader,
|
||||
) error {
|
||||
defaultStorage := store.GetDefaultStorage(instanceStorageSetting)
|
||||
if defaultStorage == nil {
|
||||
|
|
@ -99,9 +106,24 @@ func saveAttachmentBlobWithInstanceStorageSetting(
|
|||
return errors.Wrap(err, "Failed to create directory")
|
||||
}
|
||||
|
||||
// Write the blob to the file.
|
||||
if err := os.WriteFile(osPath, create.Blob, 0644); err != nil {
|
||||
return errors.Wrap(err, "Failed to write file")
|
||||
// Stage in a temp file so partial content never appears at the final path.
|
||||
file, err := os.CreateTemp(dir, ".memos-upload-*")
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to create attachment file")
|
||||
}
|
||||
defer os.Remove(file.Name())
|
||||
defer file.Close()
|
||||
if _, err := io.Copy(file, &attachmentContextReader{ctx: ctx, reader: content}); err != nil {
|
||||
return errors.Wrap(err, "failed to write attachment file")
|
||||
}
|
||||
if err := file.Chmod(0644); err != nil {
|
||||
return errors.Wrap(err, "failed to set attachment permissions")
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
return errors.Wrap(err, "failed to close attachment file")
|
||||
}
|
||||
if err := os.Rename(file.Name(), osPath); err != nil {
|
||||
return errors.Wrap(err, "failed to finalize attachment file")
|
||||
}
|
||||
create.Reference = internalPath
|
||||
create.Blob = nil
|
||||
|
|
@ -117,7 +139,7 @@ func saveAttachmentBlobWithInstanceStorageSetting(
|
|||
filepathTemplate = filepath.Join(filepathTemplate, "{filename}")
|
||||
}
|
||||
filepathTemplate = replaceFilenameWithPathTemplate(filepathTemplate, create.Filename)
|
||||
key, err := driver.UploadObject(ctx, filepathTemplate, create.Type, bytes.NewReader(create.Blob))
|
||||
key, err := driver.UploadObject(ctx, filepathTemplate, create.Type, content)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to upload via storage driver")
|
||||
}
|
||||
|
|
@ -133,6 +155,12 @@ func saveAttachmentBlobWithInstanceStorageSetting(
|
|||
},
|
||||
}
|
||||
create.Payload = payload
|
||||
} else {
|
||||
blob, err := io.ReadAll(content)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to read attachment content")
|
||||
}
|
||||
create.Blob = blob
|
||||
}
|
||||
|
||||
return nil
|
||||
|
|
|
|||
214
server/router/api/v1/attachment_upload.go
Normal file
214
server/router/api/v1/attachment_upload.go
Normal file
|
|
@ -0,0 +1,214 @@
|
|||
package v1
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"io"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
v1pb "github.com/usememos/memos/proto/gen/api/v1"
|
||||
"github.com/usememos/memos/store"
|
||||
)
|
||||
|
||||
// UploadAttachment accepts bounded unary chunks. A call carrying a spec opens
|
||||
// a new upload; a call carrying an upload ID continues one. Either kind may
|
||||
// write data and finalize.
|
||||
func (s *APIV1Service) UploadAttachment(ctx context.Context, request *v1pb.UploadAttachmentRequest) (*v1pb.UploadAttachmentResponse, error) {
|
||||
user, err := s.requireCurrentSpaceUser(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(request.Data) > attachmentUploadChunkSize {
|
||||
return nil, status.Errorf(codes.ResourceExhausted, "upload chunk exceeds the limit")
|
||||
}
|
||||
var id string
|
||||
var upload *attachmentUpload
|
||||
switch u := request.Upload.(type) {
|
||||
case *v1pb.UploadAttachmentRequest_Spec:
|
||||
id, upload, err = s.startAttachmentUpload(ctx, request, u.Spec, user.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
upload.mu.Lock()
|
||||
case *v1pb.UploadAttachmentRequest_UploadId:
|
||||
id = u.UploadId
|
||||
upload, err = s.attachmentUploads.get(id, user.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
upload.mu.Lock()
|
||||
if !time.Now().Before(upload.expireTime) {
|
||||
upload.mu.Unlock()
|
||||
return nil, status.Errorf(codes.NotFound, "upload not found or expired")
|
||||
}
|
||||
default:
|
||||
return nil, status.Errorf(codes.InvalidArgument, "spec or upload_id is required")
|
||||
}
|
||||
defer upload.mu.Unlock()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, status.FromContextError(err).Err()
|
||||
}
|
||||
if err := upload.write(ctx, request); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var attachment *v1pb.Attachment
|
||||
if upload.complete {
|
||||
// Recheck access and existence; a completed upload must not resurrect a
|
||||
// deleted attachment or disclose metadata after access is revoked.
|
||||
attachment, err = s.GetAttachment(ctx, &v1pb.GetAttachmentRequest{Name: "attachments/" + upload.uid})
|
||||
} else if request.FinishWrite {
|
||||
if upload.committedSize != upload.totalSize {
|
||||
return nil, status.Errorf(codes.FailedPrecondition, "upload is incomplete")
|
||||
}
|
||||
attachment, err = s.finishAttachmentUpload(ctx, upload)
|
||||
if err == nil {
|
||||
upload.complete = true
|
||||
os.Remove(upload.path)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
upload.expireTime = time.Now().Add(attachmentUploadTTL)
|
||||
return &v1pb.UploadAttachmentResponse{
|
||||
UploadId: id, CommittedSize: upload.committedSize, Attachment: attachment, MaxChunkSize: attachmentUploadChunkSize,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// startAttachmentUpload validates the spec and registers a new upload. Every
|
||||
// check that can fail on the accompanying data runs first, so a rejected call
|
||||
// never leaves an orphaned upload behind.
|
||||
func (s *APIV1Service) startAttachmentUpload(ctx context.Context, request *v1pb.UploadAttachmentRequest, spec *v1pb.UploadAttachmentSpec, ownerID int32) (string, *attachmentUpload, error) {
|
||||
if spec.Attachment == nil {
|
||||
return "", nil, status.Errorf(codes.InvalidArgument, "spec.attachment is required")
|
||||
}
|
||||
if len(spec.Attachment.Content) != 0 {
|
||||
return "", nil, status.Errorf(codes.InvalidArgument, "spec.attachment.content must be empty; send file bytes in data")
|
||||
}
|
||||
if proto.Size(spec) > attachmentUploadMetadataLimit || spec.TotalSize < 0 {
|
||||
return "", nil, status.Errorf(codes.InvalidArgument, "invalid attachment metadata or total_size")
|
||||
}
|
||||
if request.WriteOffset != 0 {
|
||||
return "", nil, status.Errorf(codes.OutOfRange, "write_offset must be 0 for a new upload")
|
||||
}
|
||||
if int64(len(request.Data)) > spec.TotalSize {
|
||||
return "", nil, status.Errorf(codes.InvalidArgument, "data exceeds total_size")
|
||||
}
|
||||
setting, err := s.Store.GetInstanceStorageSetting(ctx)
|
||||
if err != nil {
|
||||
return "", nil, status.Errorf(codes.Internal, "failed to get storage setting: %v", err)
|
||||
}
|
||||
if err := checkUploadSize(setting, spec.TotalSize); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
// Validate before allocating a temporary file. Keep the original MIME type
|
||||
// so an omitted type can be sniffed from real bytes at finalization.
|
||||
metadata := proto.CloneOf(spec.Attachment)
|
||||
validationMetadata := proto.CloneOf(metadata)
|
||||
if validationMetadata.Type == "" {
|
||||
// Type-dependent metadata validation must wait for content sniffing.
|
||||
validationMetadata.MediaMetadata = nil
|
||||
}
|
||||
create, err := s.prepareAttachment(ctx, &v1pb.CreateAttachmentRequest{Attachment: validationMetadata, AttachmentId: spec.AttachmentId})
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
upload := &attachmentUpload{ownerID: ownerID, metadata: metadata, uid: create.UID, totalSize: spec.TotalSize}
|
||||
id, err := s.attachmentUploads.create(s.Profile.Data, upload)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
return id, upload, nil
|
||||
}
|
||||
|
||||
func (u *attachmentUpload) write(ctx context.Context, request *v1pb.UploadAttachmentRequest) error {
|
||||
if len(request.Data) == 0 {
|
||||
if request.FinishWrite && request.WriteOffset != u.committedSize {
|
||||
return status.Errorf(codes.OutOfRange, "write_offset must equal committed_size")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
digest := sha256.Sum256(request.Data)
|
||||
if u.committedSize > 0 && request.WriteOffset+int64(len(request.Data)) == u.committedSize && digest == u.lastDigest {
|
||||
return nil // A lost response can be retried without appending bytes twice.
|
||||
}
|
||||
if u.complete {
|
||||
return status.Errorf(codes.FailedPrecondition, "upload is already complete")
|
||||
}
|
||||
if request.WriteOffset != u.committedSize {
|
||||
return status.Errorf(codes.OutOfRange, "write_offset must equal committed_size")
|
||||
}
|
||||
if int64(len(request.Data)) > u.totalSize-u.committedSize {
|
||||
return status.Errorf(codes.InvalidArgument, "data exceeds total_size")
|
||||
}
|
||||
file, err := os.OpenFile(u.path, os.O_WRONLY, 0600)
|
||||
if err != nil {
|
||||
return status.Errorf(codes.Internal, "failed to open upload file: %v", err)
|
||||
}
|
||||
defer file.Close()
|
||||
n, err := file.WriteAt(request.Data, u.committedSize)
|
||||
if err == nil && n != len(request.Data) {
|
||||
err = io.ErrShortWrite
|
||||
}
|
||||
if err == nil {
|
||||
err = ctx.Err()
|
||||
}
|
||||
if err != nil {
|
||||
if truncateErr := file.Truncate(u.committedSize); truncateErr != nil {
|
||||
// The file no longer matches committedSize; revoke the upload.
|
||||
u.expireTime = time.Time{}
|
||||
os.Remove(u.path)
|
||||
}
|
||||
return status.Errorf(codes.Internal, "failed to write upload chunk: %v", err)
|
||||
}
|
||||
u.lastDigest = digest
|
||||
u.committedSize += int64(n)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *APIV1Service) finishAttachmentUpload(ctx context.Context, upload *attachmentUpload) (*v1pb.Attachment, error) {
|
||||
if upload.finalizeAttempted {
|
||||
// A database operation may commit before returning an error. Resolve that
|
||||
// outcome before saving another object under the upload's stable UID.
|
||||
persisted, err := s.Store.GetAttachment(ctx, &store.FindAttachment{UID: &upload.uid})
|
||||
if err != nil {
|
||||
return nil, status.Errorf(codes.Internal, "failed to resolve previous finalization: %v", err)
|
||||
}
|
||||
if persisted != nil {
|
||||
if persisted.CreatorID != upload.ownerID {
|
||||
return nil, status.Errorf(codes.AlreadyExists, "attachment ID already exists")
|
||||
}
|
||||
return s.GetAttachment(ctx, &v1pb.GetAttachmentRequest{Name: "attachments/" + upload.uid})
|
||||
}
|
||||
}
|
||||
setting, err := s.Store.GetInstanceStorageSetting(ctx)
|
||||
if err != nil {
|
||||
return nil, status.Errorf(codes.Internal, "failed to get storage setting: %v", err)
|
||||
}
|
||||
if err := checkUploadSize(setting, upload.totalSize); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
file, err := os.Open(upload.path)
|
||||
if err != nil {
|
||||
return nil, status.Errorf(codes.Internal, "failed to open upload file: %v", err)
|
||||
}
|
||||
defer file.Close()
|
||||
metadata := proto.CloneOf(upload.metadata)
|
||||
metadata.Content = make([]byte, min(upload.totalSize, 512))
|
||||
if _, err := io.ReadFull(file, metadata.Content); err != nil {
|
||||
return nil, status.Errorf(codes.Internal, "failed to read attachment header: %v", err)
|
||||
}
|
||||
// Revalidate memo permissions and media metadata with the actual MIME type.
|
||||
create, err := s.prepareAttachment(ctx, &v1pb.CreateAttachmentRequest{Attachment: metadata, AttachmentId: upload.uid})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
create.Size = upload.totalSize
|
||||
upload.finalizeAttempted = true
|
||||
return s.processAndSaveAttachment(ctx, create, setting, file)
|
||||
}
|
||||
194
server/router/api/v1/attachment_upload_state.go
Normal file
194
server/router/api/v1/attachment_upload_state.go
Normal file
|
|
@ -0,0 +1,194 @@
|
|||
package v1
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
|
||||
v1pb "github.com/usememos/memos/proto/gen/api/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
attachmentUploadChunkSize = 2 << 20
|
||||
attachmentUploadRequestLimit = 4 << 20
|
||||
attachmentUploadMetadataLimit = 64 << 10
|
||||
attachmentUploadTTL = 30 * time.Minute
|
||||
attachmentUploadMaxSessions = 1024
|
||||
attachmentUploadMaxActive = 128
|
||||
attachmentUploadMaxActivePerUser = 8
|
||||
attachmentUploadTempPrefix = ".memos-rpc-upload-"
|
||||
attachmentUploadProcedure = "/memos.api.v1.AttachmentService/UploadAttachment"
|
||||
)
|
||||
|
||||
// Upload state is private to the process; only Attachment is an API resource.
|
||||
// Each upload holds metadata and the last chunk's digest, never its contents.
|
||||
type attachmentUpload struct {
|
||||
mu sync.Mutex
|
||||
ownerID int32
|
||||
metadata *v1pb.Attachment
|
||||
uid string
|
||||
path string
|
||||
totalSize int64
|
||||
committedSize int64
|
||||
// expireTime is zeroed to revoke an upload before its TTL elapses.
|
||||
expireTime time.Time
|
||||
// lastDigest identifies the most recently accepted chunk so a client can
|
||||
// safely resend it after losing the response.
|
||||
lastDigest [sha256.Size]byte
|
||||
finalizeAttempted bool
|
||||
complete bool
|
||||
}
|
||||
|
||||
type attachmentUploads struct {
|
||||
mu sync.Mutex
|
||||
entries map[string]*attachmentUpload
|
||||
stop chan struct{}
|
||||
done chan struct{}
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (m *attachmentUploads) startLocked(dir string) {
|
||||
if m.entries != nil {
|
||||
return
|
||||
}
|
||||
m.entries = make(map[string]*attachmentUpload)
|
||||
m.stop = make(chan struct{})
|
||||
m.done = make(chan struct{})
|
||||
go func() {
|
||||
defer close(m.done)
|
||||
ticker := time.NewTicker(time.Minute)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-m.stop:
|
||||
return
|
||||
case now := <-ticker.C:
|
||||
m.mu.Lock()
|
||||
m.sweepLocked(now, 0)
|
||||
m.removeOrphansLocked(dir, now)
|
||||
m.mu.Unlock()
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// sweepLocked drops expired uploads and counts the unfinished ones, both in
|
||||
// total and for ownerID. An upload locked by an in-flight request is
|
||||
// unfinished by definition.
|
||||
func (m *attachmentUploads) sweepLocked(now time.Time, ownerID int32) (owned, total int) {
|
||||
count := func(upload *attachmentUpload) {
|
||||
total++
|
||||
if upload.ownerID == ownerID {
|
||||
owned++
|
||||
}
|
||||
}
|
||||
for id, upload := range m.entries {
|
||||
// Never hold the manager lock while waiting for a file write or finalize.
|
||||
if !upload.mu.TryLock() {
|
||||
count(upload)
|
||||
continue
|
||||
}
|
||||
if !now.Before(upload.expireTime) {
|
||||
os.Remove(upload.path)
|
||||
delete(m.entries, id)
|
||||
} else if !upload.complete {
|
||||
count(upload)
|
||||
}
|
||||
upload.mu.Unlock()
|
||||
}
|
||||
return owned, total
|
||||
}
|
||||
|
||||
func (m *attachmentUploads) removeOrphansLocked(dir string, now time.Time) {
|
||||
tracked := make(map[string]bool, len(m.entries))
|
||||
for _, upload := range m.entries {
|
||||
tracked[upload.path] = true // path is immutable after insertion.
|
||||
}
|
||||
paths, _ := filepath.Glob(filepath.Join(dir, attachmentUploadTempPrefix+"*"))
|
||||
for _, path := range paths {
|
||||
if tracked[path] {
|
||||
continue
|
||||
}
|
||||
if info, err := os.Stat(path); err == nil && info.Mode().IsRegular() && now.Sub(info.ModTime()) >= attachmentUploadTTL {
|
||||
os.Remove(path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *attachmentUploads) create(dir string, upload *attachmentUpload) (string, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.closed {
|
||||
return "", status.Errorf(codes.Unavailable, "server is shutting down")
|
||||
}
|
||||
m.startLocked(dir)
|
||||
owned, total := m.sweepLocked(time.Now(), upload.ownerID)
|
||||
if owned >= attachmentUploadMaxActivePerUser {
|
||||
return "", status.Errorf(codes.ResourceExhausted, "too many active uploads")
|
||||
}
|
||||
if len(m.entries) >= attachmentUploadMaxSessions || total >= attachmentUploadMaxActive {
|
||||
return "", status.Errorf(codes.ResourceExhausted, "too many uploads")
|
||||
}
|
||||
file, err := os.CreateTemp(dir, attachmentUploadTempPrefix+"*")
|
||||
if err != nil {
|
||||
return "", status.Errorf(codes.Internal, "failed to create upload file: %v", err)
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
os.Remove(file.Name())
|
||||
return "", status.Errorf(codes.Internal, "failed to close upload file: %v", err)
|
||||
}
|
||||
upload.path = file.Name()
|
||||
upload.expireTime = time.Now().Add(attachmentUploadTTL)
|
||||
id := rand.Text()
|
||||
m.entries[id] = upload
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (m *attachmentUploads) get(id string, ownerID int32) (*attachmentUpload, error) {
|
||||
m.mu.Lock()
|
||||
upload := m.entries[id]
|
||||
closed := m.closed
|
||||
m.mu.Unlock()
|
||||
if closed {
|
||||
return nil, status.Errorf(codes.Unavailable, "server is shutting down")
|
||||
}
|
||||
if upload == nil || upload.ownerID != ownerID {
|
||||
return nil, status.Errorf(codes.NotFound, "upload not found or expired")
|
||||
}
|
||||
return upload, nil
|
||||
}
|
||||
|
||||
// CloseAttachmentUploads stops expiration work and removes pending upload files.
|
||||
// Call it after draining HTTP requests during server shutdown.
|
||||
func (s *APIV1Service) CloseAttachmentUploads() {
|
||||
m := &s.attachmentUploads
|
||||
m.mu.Lock()
|
||||
if m.closed {
|
||||
m.mu.Unlock()
|
||||
return
|
||||
}
|
||||
m.closed = true
|
||||
if m.stop != nil {
|
||||
close(m.stop)
|
||||
}
|
||||
done := m.done
|
||||
m.mu.Unlock()
|
||||
if done != nil {
|
||||
<-done
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
for id, upload := range m.entries {
|
||||
upload.mu.Lock()
|
||||
upload.expireTime = time.Time{}
|
||||
os.Remove(upload.path)
|
||||
upload.mu.Unlock()
|
||||
delete(m.entries, id)
|
||||
}
|
||||
}
|
||||
357
server/router/api/v1/attachment_upload_test.go
Normal file
357
server/router/api/v1/attachment_upload_test.go
Normal file
|
|
@ -0,0 +1,357 @@
|
|||
package v1
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/usememos/memos/internal/testutil"
|
||||
v1pb "github.com/usememos/memos/proto/gen/api/v1"
|
||||
storepb "github.com/usememos/memos/proto/gen/store"
|
||||
"github.com/usememos/memos/store"
|
||||
)
|
||||
|
||||
func newUploadTestService(t *testing.T) (*APIV1Service, context.Context) {
|
||||
t.Helper()
|
||||
svc := newIntegrationService(t)
|
||||
t.Cleanup(svc.CloseAttachmentUploads)
|
||||
user := createSpaceTestUser(context.Background(), t, svc, "uploader", store.RoleUser)
|
||||
return svc, userCtx(context.Background(), user.ID)
|
||||
}
|
||||
|
||||
func uploadSpec(filename string, size int64) *v1pb.UploadAttachmentRequest_Spec {
|
||||
return &v1pb.UploadAttachmentRequest_Spec{Spec: &v1pb.UploadAttachmentSpec{Attachment: &v1pb.Attachment{Filename: filename}, TotalSize: size}}
|
||||
}
|
||||
|
||||
func uploadID(id string) *v1pb.UploadAttachmentRequest_UploadId {
|
||||
return &v1pb.UploadAttachmentRequest_UploadId{UploadId: id}
|
||||
}
|
||||
|
||||
func startTestUpload(ctx context.Context, t *testing.T, svc *APIV1Service, size int64) string {
|
||||
t.Helper()
|
||||
spec := uploadSpec("file.bin", size)
|
||||
spec.Spec.Attachment.Type = "application/octet-stream"
|
||||
response, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: spec})
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, response.UploadId)
|
||||
require.EqualValues(t, attachmentUploadChunkSize, response.MaxChunkSize)
|
||||
require.Zero(t, response.CommittedSize)
|
||||
require.Nil(t, response.Attachment)
|
||||
return response.UploadId
|
||||
}
|
||||
|
||||
func TestUploadAttachmentChunksAndRetries(t *testing.T) {
|
||||
svc, ctx := newUploadTestService(t)
|
||||
id := startTestUpload(ctx, t, svc, 6)
|
||||
first := &v1pb.UploadAttachmentRequest{Upload: uploadID(id), Data: []byte("abc")}
|
||||
for range 2 {
|
||||
response, err := svc.UploadAttachment(ctx, first)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, id, response.UploadId)
|
||||
require.EqualValues(t, 3, response.CommittedSize)
|
||||
}
|
||||
for _, request := range []*v1pb.UploadAttachmentRequest{
|
||||
{Upload: uploadID(id), Data: []byte("xyz")},
|
||||
{Upload: uploadID(id), WriteOffset: 4, Data: []byte("d")},
|
||||
{Upload: uploadID(id), WriteOffset: -1, Data: []byte("d")},
|
||||
} {
|
||||
_, err := svc.UploadAttachment(ctx, request)
|
||||
require.Equal(t, codes.OutOfRange, status.Code(err))
|
||||
}
|
||||
progress, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(id), WriteOffset: -1})
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, 3, progress.CommittedSize)
|
||||
_, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(id), WriteOffset: 3, FinishWrite: true})
|
||||
require.Equal(t, codes.FailedPrecondition, status.Code(err))
|
||||
|
||||
last := &v1pb.UploadAttachmentRequest{Upload: uploadID(id), WriteOffset: 3, Data: []byte("def"), FinishWrite: true}
|
||||
result, err := svc.UploadAttachment(ctx, last)
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, 6, result.Attachment.Size)
|
||||
for _, request := range []*v1pb.UploadAttachmentRequest{last, {Upload: uploadID(id)}, {Upload: uploadID(id), WriteOffset: 6, FinishWrite: true}} {
|
||||
repeated, err := svc.UploadAttachment(ctx, request)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, result.Attachment.Name, repeated.Attachment.Name)
|
||||
}
|
||||
rows, err := svc.Store.ListAttachments(ctx, &store.FindAttachment{})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 1)
|
||||
content, err := os.ReadFile(rows[0].Reference)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "abcdef", string(content))
|
||||
require.NoFileExists(t, svc.attachmentUploads.entries[id].path)
|
||||
|
||||
_, err = svc.DeleteAttachment(ctx, &v1pb.DeleteAttachmentRequest{Name: result.Attachment.Name})
|
||||
require.NoError(t, err)
|
||||
_, err = svc.UploadAttachment(ctx, last)
|
||||
require.Equal(t, codes.NotFound, status.Code(err))
|
||||
}
|
||||
|
||||
func TestUploadAttachmentSingleCall(t *testing.T) {
|
||||
svc, ctx := newUploadTestService(t)
|
||||
spec := uploadSpec("note.txt", 3)
|
||||
spec.Spec.AttachmentId = "chosen-id"
|
||||
result, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: spec, Data: []byte("abc"), FinishWrite: true})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "attachments/chosen-id", result.Attachment.Name)
|
||||
require.EqualValues(t, 3, result.Attachment.Size)
|
||||
require.EqualValues(t, 3, result.CommittedSize)
|
||||
|
||||
empty, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadSpec("empty.txt", 0), FinishWrite: true})
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, empty.Attachment.Size)
|
||||
|
||||
// A spec call may carry the first chunk without finishing.
|
||||
partial, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadSpec("two.bin", 4), Data: []byte("ab")})
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, partial.Attachment)
|
||||
require.EqualValues(t, 2, partial.CommittedSize)
|
||||
done, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(partial.UploadId), WriteOffset: 2, Data: []byte("cd"), FinishWrite: true})
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, 4, done.Attachment.Size)
|
||||
|
||||
rows, err := svc.Store.ListAttachments(ctx, &store.FindAttachment{})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 3)
|
||||
}
|
||||
|
||||
func TestUploadAttachmentValidation(t *testing.T) {
|
||||
svc, ctx := newUploadTestService(t)
|
||||
_, err := svc.UploadAttachment(context.Background(), &v1pb.UploadAttachmentRequest{})
|
||||
require.Equal(t, codes.Unauthenticated, status.Code(err))
|
||||
badID := uploadSpec("file", 0)
|
||||
badID.Spec.AttachmentId = "bad id!"
|
||||
withContent := uploadSpec("file", 0)
|
||||
withContent.Spec.Attachment.Content = []byte("not metadata")
|
||||
for _, request := range []*v1pb.UploadAttachmentRequest{
|
||||
{},
|
||||
{Upload: &v1pb.UploadAttachmentRequest_Spec{Spec: &v1pb.UploadAttachmentSpec{}}},
|
||||
{Upload: uploadSpec("file", -1)},
|
||||
{Upload: uploadSpec("../file", 0)},
|
||||
{Upload: withContent},
|
||||
{Upload: badID},
|
||||
{Upload: uploadSpec(strings.Repeat("x", attachmentUploadMetadataLimit+1), 0)},
|
||||
{Upload: uploadSpec("file", 3), Data: []byte("four")},
|
||||
} {
|
||||
_, err := svc.UploadAttachment(ctx, request)
|
||||
require.Equal(t, codes.InvalidArgument, status.Code(err), request.String())
|
||||
}
|
||||
_, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadSpec("file", 3), WriteOffset: 1, Data: []byte("abc")})
|
||||
require.Equal(t, codes.OutOfRange, status.Code(err))
|
||||
_, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadSpec("file", 1<<60)})
|
||||
require.Equal(t, codes.ResourceExhausted, status.Code(err))
|
||||
require.Empty(t, svc.attachmentUploads.entries, "rejected spec calls must not register uploads")
|
||||
|
||||
id := startTestUpload(ctx, t, svc, 3)
|
||||
other := createSpaceTestUser(context.Background(), t, svc, "other-uploader", store.RoleUser)
|
||||
_, err = svc.UploadAttachment(userCtx(context.Background(), other.ID), &v1pb.UploadAttachmentRequest{Upload: uploadID(id)})
|
||||
require.Equal(t, codes.NotFound, status.Code(err))
|
||||
_, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID("unknown")})
|
||||
require.Equal(t, codes.NotFound, status.Code(err))
|
||||
_, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(id), Data: make([]byte, attachmentUploadChunkSize+1)})
|
||||
require.Equal(t, codes.ResourceExhausted, status.Code(err))
|
||||
_, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(id), Data: []byte("four")})
|
||||
require.Equal(t, codes.InvalidArgument, status.Code(err))
|
||||
canceled, cancel := context.WithCancel(ctx)
|
||||
cancel()
|
||||
_, err = svc.UploadAttachment(canceled, &v1pb.UploadAttachmentRequest{Upload: uploadID(id), Data: []byte("abc")})
|
||||
require.Error(t, err)
|
||||
progress, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(id)})
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, progress.CommittedSize)
|
||||
}
|
||||
|
||||
func TestUploadAttachmentEmptyAndConcurrentFinish(t *testing.T) {
|
||||
svc, ctx := newUploadTestService(t)
|
||||
id := startTestUpload(ctx, t, svc, 0)
|
||||
var wg sync.WaitGroup
|
||||
results := make(chan *v1pb.UploadAttachmentResponse, 8)
|
||||
errs := make(chan error, 8)
|
||||
for range 8 {
|
||||
wg.Go(func() {
|
||||
response, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(id), FinishWrite: true})
|
||||
results <- response
|
||||
errs <- err
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
close(results)
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
name := ""
|
||||
for response := range results {
|
||||
if name == "" {
|
||||
name = response.Attachment.Name
|
||||
}
|
||||
require.Equal(t, name, response.Attachment.Name)
|
||||
require.Zero(t, response.Attachment.Size)
|
||||
}
|
||||
rows, err := svc.Store.ListAttachments(ctx, &store.FindAttachment{})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 1)
|
||||
}
|
||||
|
||||
func TestUploadAttachmentFinalizationRecovery(t *testing.T) {
|
||||
svc, ctx := newUploadTestService(t)
|
||||
id := startTestUpload(ctx, t, svc, 3)
|
||||
last := &v1pb.UploadAttachmentRequest{Upload: uploadID(id), Data: []byte("abc"), FinishWrite: true}
|
||||
_, err := svc.UploadAttachment(store.WithCreateAttachmentPostCommitFailpoint(ctx), last)
|
||||
require.Equal(t, codes.Internal, status.Code(err))
|
||||
result, err := svc.UploadAttachment(ctx, last)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result.Attachment)
|
||||
rows, err := svc.Store.ListAttachments(ctx, &store.FindAttachment{})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 1)
|
||||
content, err := os.ReadFile(rows[0].Reference)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "abc", string(content))
|
||||
}
|
||||
|
||||
func TestUploadAttachmentRechecksMemoAndSize(t *testing.T) {
|
||||
svc, ctx := newUploadTestService(t)
|
||||
memo, err := svc.CreateMemo(ctx, &v1pb.CreateMemoRequest{Memo: &v1pb.Memo{Content: "upload target"}})
|
||||
require.NoError(t, err)
|
||||
spec := uploadSpec("file.txt", 3)
|
||||
spec.Spec.Attachment.Memo = &memo.Name
|
||||
initial, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: spec})
|
||||
require.NoError(t, err)
|
||||
_, err = svc.DeleteMemo(ctx, &v1pb.DeleteMemoRequest{Name: memo.Name, Force: true})
|
||||
require.NoError(t, err)
|
||||
_, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(initial.UploadId), Data: []byte("abc"), FinishWrite: true})
|
||||
require.Equal(t, codes.NotFound, status.Code(err))
|
||||
|
||||
id := startTestUpload(ctx, t, svc, 2*MebiByte)
|
||||
_, err = svc.Store.UpsertInstanceSetting(ctx, &storepb.InstanceSetting{Key: storepb.InstanceSettingKey_STORAGE,
|
||||
Value: &storepb.InstanceSetting_StorageSetting{StorageSetting: &storepb.InstanceStorageSetting{UploadSizeLimitMb: 1}},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
_, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(id), Data: make([]byte, 2*MebiByte), FinishWrite: true})
|
||||
require.Equal(t, codes.ResourceExhausted, status.Code(err))
|
||||
rows, err := svc.Store.ListAttachments(ctx, &store.FindAttachment{})
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, rows)
|
||||
}
|
||||
|
||||
func TestUploadAttachmentExpiryAndLimits(t *testing.T) {
|
||||
svc, ctx := newUploadTestService(t)
|
||||
ids := make([]string, attachmentUploadMaxActivePerUser)
|
||||
for i := range ids {
|
||||
ids[i] = startTestUpload(ctx, t, svc, 0)
|
||||
}
|
||||
_, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadSpec("too-many", 0)})
|
||||
require.Equal(t, codes.ResourceExhausted, status.Code(err))
|
||||
_, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(ids[0]), FinishWrite: true})
|
||||
require.NoError(t, err)
|
||||
startTestUpload(ctx, t, svc, 0) // Completed uploads do not consume an active slot.
|
||||
upload := svc.attachmentUploads.entries[ids[1]]
|
||||
upload.mu.Lock()
|
||||
upload.expireTime = time.Now().Add(-time.Second)
|
||||
upload.mu.Unlock()
|
||||
_, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(ids[1])})
|
||||
require.Equal(t, codes.NotFound, status.Code(err))
|
||||
|
||||
orphan := filepath.Join(svc.Profile.Data, attachmentUploadTempPrefix+"orphan")
|
||||
recent := filepath.Join(svc.Profile.Data, attachmentUploadTempPrefix+"recent")
|
||||
unrelated := filepath.Join(svc.Profile.Data, "keep.txt")
|
||||
for _, path := range []string{orphan, recent, unrelated} {
|
||||
require.NoError(t, os.WriteFile(path, []byte("x"), 0600))
|
||||
}
|
||||
old := time.Now().Add(-2 * attachmentUploadTTL)
|
||||
require.NoError(t, os.Chtimes(orphan, old, old))
|
||||
require.NoError(t, os.Chtimes(unrelated, old, old))
|
||||
svc.attachmentUploads.mu.Lock()
|
||||
svc.attachmentUploads.sweepLocked(time.Now(), 0)
|
||||
svc.attachmentUploads.removeOrphansLocked(svc.Profile.Data, time.Now())
|
||||
svc.attachmentUploads.mu.Unlock()
|
||||
require.NoFileExists(t, upload.path)
|
||||
require.NoFileExists(t, orphan)
|
||||
require.FileExists(t, recent)
|
||||
require.FileExists(t, unrelated)
|
||||
svc.CloseAttachmentUploads()
|
||||
require.Empty(t, svc.attachmentUploads.entries)
|
||||
}
|
||||
|
||||
func TestUploadAttachmentAboveLegacyRequestLimit(t *testing.T) {
|
||||
svc, ctx := newUploadTestService(t)
|
||||
const size = 300 * MebiByte
|
||||
_, err := svc.Store.UpsertInstanceSetting(ctx, &storepb.InstanceSetting{Key: storepb.InstanceSettingKey_STORAGE,
|
||||
Value: &storepb.InstanceSetting_StorageSetting{StorageSetting: &storepb.InstanceStorageSetting{UploadSizeLimitMb: 400}},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
id := startTestUpload(ctx, t, svc, size)
|
||||
chunk := bytes.Repeat([]byte{0x5a}, attachmentUploadChunkSize)
|
||||
var response *v1pb.UploadAttachmentResponse
|
||||
for offset := int64(0); offset < size; offset += int64(len(chunk)) {
|
||||
response, err = svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{
|
||||
Upload: uploadID(id), WriteOffset: offset, Data: chunk, FinishWrite: offset+int64(len(chunk)) == size,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.EqualValues(t, size, response.Attachment.Size)
|
||||
rows, err := svc.Store.ListAttachments(ctx, &store.FindAttachment{})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 1)
|
||||
file, err := os.Open(rows[0].Reference)
|
||||
require.NoError(t, err)
|
||||
defer file.Close()
|
||||
info, err := file.Stat()
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, size, info.Size())
|
||||
_, err = file.Seek(-int64(len(chunk)), io.SeekEnd)
|
||||
require.NoError(t, err)
|
||||
tail := make([]byte, len(chunk))
|
||||
_, err = io.ReadFull(file, tail)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, chunk, tail)
|
||||
}
|
||||
|
||||
func TestUploadAttachmentMediaProcessing(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
content []byte
|
||||
motion bool
|
||||
}{
|
||||
{name: "JPEG", content: testutil.BuildJPEG(20, 10)},
|
||||
{name: "motion photo", content: testutil.BuildMotionPhotoJPEG(), motion: true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
svc, ctx := newUploadTestService(t)
|
||||
metadata := &v1pb.Attachment{Filename: "image.unknown", MediaMetadata: &v1pb.MediaMetadata{Width: proto.Int32(20), Height: proto.Int32(10)}}
|
||||
spec := &v1pb.UploadAttachmentRequest_Spec{Spec: &v1pb.UploadAttachmentSpec{Attachment: metadata, TotalSize: int64(len(tc.content))}}
|
||||
initial, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: spec})
|
||||
require.NoError(t, err)
|
||||
result, err := svc.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: uploadID(initial.UploadId), Data: tc.content, FinishWrite: true})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "image/jpeg", result.Attachment.Type)
|
||||
require.True(t, proto.Equal(metadata.MediaMetadata, result.Attachment.MediaMetadata))
|
||||
uid, err := ExtractAttachmentUIDFromName(result.Attachment.Name)
|
||||
require.NoError(t, err)
|
||||
row, err := svc.Store.GetAttachment(ctx, &store.FindAttachment{UID: &uid})
|
||||
require.NoError(t, err)
|
||||
content, err := os.ReadFile(row.Reference)
|
||||
require.NoError(t, err)
|
||||
if tc.motion {
|
||||
require.Equal(t, tc.content, content)
|
||||
require.True(t, result.Attachment.MotionMedia.HasEmbeddedVideo)
|
||||
} else {
|
||||
stripped, err := stripImageExif(bytes.NewReader(tc.content), "image/jpeg")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, stripped, content)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
111
server/router/api/v1/attachment_upload_transport_test.go
Normal file
111
server/router/api/v1/attachment_upload_transport_test.go
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
package v1
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"connectrpc.com/connect"
|
||||
"github.com/labstack/echo/v5"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/protobuf/encoding/protojson"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
v1pb "github.com/usememos/memos/proto/gen/api/v1"
|
||||
"github.com/usememos/memos/proto/gen/api/v1/apiv1connect"
|
||||
"github.com/usememos/memos/server/auth"
|
||||
)
|
||||
|
||||
type uploadCountingReader struct {
|
||||
reader io.Reader
|
||||
read int
|
||||
}
|
||||
|
||||
func (r *uploadCountingReader) Read(p []byte) (int, error) {
|
||||
n, err := r.reader.Read(p)
|
||||
r.read += n
|
||||
return n, err
|
||||
}
|
||||
|
||||
func TestUploadAttachmentTransports(t *testing.T) {
|
||||
svc, ctx := newUploadTestService(t)
|
||||
user, err := svc.fetchCurrentUser(ctx)
|
||||
require.NoError(t, err)
|
||||
token, _, err := auth.GenerateAccessTokenV2(user.ID, user.Username, string(user.Role), string(user.RowStatus), []byte(svc.Secret))
|
||||
require.NoError(t, err)
|
||||
e := echo.New()
|
||||
require.NoError(t, svc.RegisterGateway(context.Background(), e))
|
||||
|
||||
for _, path := range []string{"/api/v1/attachments:upload", attachmentUploadProcedure} {
|
||||
t.Run(path, func(t *testing.T) {
|
||||
call := func(message proto.Message, authenticated bool) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
data, err := protojson.Marshal(message)
|
||||
require.NoError(t, err)
|
||||
req := httptest.NewRequest(http.MethodPost, path, bytes.NewReader(data))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if authenticated {
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
e.ServeHTTP(rec, req)
|
||||
return rec
|
||||
}
|
||||
initial := &v1pb.UploadAttachmentRequest{Upload: uploadSpec("file.txt", 3)}
|
||||
require.Equal(t, http.StatusUnauthorized, call(initial, false).Code)
|
||||
rec := call(initial, true)
|
||||
require.Equal(t, http.StatusOK, rec.Code, rec.Body.String())
|
||||
response := &v1pb.UploadAttachmentResponse{}
|
||||
require.NoError(t, protojson.Unmarshal(rec.Body.Bytes(), response))
|
||||
rec = call(&v1pb.UploadAttachmentRequest{Upload: uploadID(response.UploadId), Data: []byte("abc"), FinishWrite: true}, true)
|
||||
require.Equal(t, http.StatusOK, rec.Code, rec.Body.String())
|
||||
require.NoError(t, protojson.Unmarshal(rec.Body.Bytes(), response))
|
||||
require.NotNil(t, response.Attachment)
|
||||
|
||||
oversized, err := protojson.Marshal(&v1pb.UploadAttachmentRequest{Data: make([]byte, attachmentUploadRequestLimit)})
|
||||
require.NoError(t, err)
|
||||
reader := &uploadCountingReader{reader: bytes.NewReader(oversized)}
|
||||
req := httptest.NewRequest(http.MethodPost, path, reader)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
rec = httptest.NewRecorder()
|
||||
e.ServeHTTP(rec, req)
|
||||
require.GreaterOrEqual(t, rec.Code, 400)
|
||||
require.LessOrEqual(t, reader.read, attachmentUploadRequestLimit+1, "body must be bounded before decoding")
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("Connect protobuf and compressed message limit", func(t *testing.T) {
|
||||
server := httptest.NewTestServer(t, e)
|
||||
client := apiv1connect.NewAttachmentServiceClient(server.Client(), server.URL)
|
||||
initial := connect.NewRequest(&v1pb.UploadAttachmentRequest{Upload: uploadSpec("binary.bin", attachmentUploadChunkSize)})
|
||||
initial.Header().Set("Authorization", "Bearer "+token)
|
||||
response, err := client.UploadAttachment(context.Background(), initial)
|
||||
require.NoError(t, err)
|
||||
last := connect.NewRequest(&v1pb.UploadAttachmentRequest{Upload: uploadID(response.Msg.UploadId), Data: make([]byte, attachmentUploadChunkSize), FinishWrite: true})
|
||||
last.Header().Set("Authorization", "Bearer "+token)
|
||||
finished, err := client.UploadAttachment(context.Background(), last)
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, attachmentUploadChunkSize, finished.Msg.Attachment.Size)
|
||||
|
||||
data, err := proto.Marshal(&v1pb.UploadAttachmentRequest{Data: make([]byte, attachmentUploadRequestLimit+1)})
|
||||
require.NoError(t, err)
|
||||
var compressed bytes.Buffer
|
||||
writer := gzip.NewWriter(&compressed)
|
||||
_, err = writer.Write(data)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, writer.Close())
|
||||
req := httptest.NewRequest(http.MethodPost, attachmentUploadProcedure, &compressed)
|
||||
req.Header.Set("Content-Type", "application/proto")
|
||||
req.Header.Set("Content-Encoding", "gzip")
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
rec := httptest.NewRecorder()
|
||||
e.ServeHTTP(rec, req)
|
||||
require.Equal(t, http.StatusTooManyRequests, rec.Code, rec.Body.String())
|
||||
require.Contains(t, rec.Body.String(), "resource_exhausted")
|
||||
})
|
||||
}
|
||||
|
|
@ -595,6 +595,14 @@ func (s *ConnectServiceHandler) DeleteSpaceMember(ctx context.Context, req *conn
|
|||
|
||||
// AttachmentService
|
||||
|
||||
func (s *ConnectServiceHandler) UploadAttachment(ctx context.Context, req *connect.Request[v1pb.UploadAttachmentRequest]) (*connect.Response[v1pb.UploadAttachmentResponse], error) {
|
||||
resp, err := s.APIV1Service.UploadAttachment(ctx, req.Msg)
|
||||
if err != nil {
|
||||
return nil, convertGRPCError(err)
|
||||
}
|
||||
return connect.NewResponse(resp), nil
|
||||
}
|
||||
|
||||
func (s *ConnectServiceHandler) CreateAttachment(ctx context.Context, req *connect.Request[v1pb.CreateAttachmentRequest]) (*connect.Response[v1pb.Attachment], error) {
|
||||
resp, err := s.APIV1Service.CreateAttachment(ctx, req.Msg)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -21,6 +21,40 @@ func TestS3AttachmentLifecycleAcrossStorageChange(t *testing.T) {
|
|||
runS3AttachmentLifecycleAcrossStorageChange(t, fake)
|
||||
}
|
||||
|
||||
func TestUploadAttachmentS3(t *testing.T) {
|
||||
ts := NewTestService(t)
|
||||
defer ts.Cleanup()
|
||||
defer ts.Service.CloseAttachmentUploads()
|
||||
fake := fakes3.New(t, "uploads")
|
||||
ctx := context.Background()
|
||||
user, err := ts.CreateRegularUser(ctx, "chunk-uploader")
|
||||
require.NoError(t, err)
|
||||
ctx = ts.CreateUserContext(ctx, user.ID)
|
||||
storage := fakeStorage("upload-s3", "Uploads", fake.Config("uploads"))
|
||||
upsertS3StorageSetting(ctx, t, ts, storage.Id, storage)
|
||||
chunk := bytes.Repeat([]byte("x"), 2<<20)
|
||||
initial, err := ts.Service.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{Upload: &v1pb.UploadAttachmentRequest_Spec{Spec: &v1pb.UploadAttachmentSpec{
|
||||
Attachment: &v1pb.Attachment{Filename: "large.bin", Type: "application/octet-stream"}, TotalSize: int64(3 * len(chunk)),
|
||||
}}})
|
||||
require.NoError(t, err)
|
||||
var result *v1pb.UploadAttachmentResponse
|
||||
for i := range 3 {
|
||||
result, err = ts.Service.UploadAttachment(ctx, &v1pb.UploadAttachmentRequest{
|
||||
Upload: &v1pb.UploadAttachmentRequest_UploadId{UploadId: initial.UploadId}, WriteOffset: int64(i * len(chunk)), Data: chunk, FinishWrite: i == 2,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
uid, err := apiv1.ExtractAttachmentUIDFromName(result.Attachment.Name)
|
||||
require.NoError(t, err)
|
||||
row, err := ts.Store.GetAttachment(ctx, &store.FindAttachment{UID: &uid})
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, row.Blob)
|
||||
require.Equal(t, storepb.AttachmentStorageType_S3, row.StorageType)
|
||||
content, err := fake.GetObject("uploads", row.Payload.GetS3Object().GetKey())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, bytes.Repeat(chunk, 3), content)
|
||||
}
|
||||
|
||||
func TestS3AttachmentLifecycleAcrossStorageChangeMinIO(t *testing.T) {
|
||||
server := testminio.New(t, "attachments-old", "attachments-new")
|
||||
runS3AttachmentLifecycleAcrossStorageChange(t, server)
|
||||
|
|
|
|||
|
|
@ -49,14 +49,14 @@ func TestUpdateMemoTimestamps(t *testing.T) {
|
|||
}})
|
||||
require.NoError(t, err)
|
||||
tc.patch.Name = memo.Name
|
||||
before := time.Now().Unix()
|
||||
beforeSec := time.Now().Unix()
|
||||
updated, err := ts.Service.UpdateMemo(userCtx, &apiv1.UpdateMemoRequest{
|
||||
Memo: tc.patch, UpdateMask: &fieldmaskpb.FieldMask{Paths: tc.paths},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tc.wantCreate, updated.CreateTime.Seconds)
|
||||
if tc.autoUpdate {
|
||||
require.GreaterOrEqual(t, updated.UpdateTime.Seconds, before)
|
||||
require.GreaterOrEqual(t, updated.UpdateTime.Seconds, beforeSec)
|
||||
require.LessOrEqual(t, updated.UpdateTime.Seconds, time.Now().Unix())
|
||||
} else {
|
||||
require.Equal(t, tc.wantUpdate, updated.UpdateTime.Seconds)
|
||||
|
|
|
|||
|
|
@ -25,6 +25,15 @@ import (
|
|||
// it derives its own limit from this constant to keep the two gates in lockstep.
|
||||
const MaxAPIRequestBytes = 256 << 20
|
||||
|
||||
// requestBodyLimit returns the request body cap for a procedure. Chunked
|
||||
// uploads carry at most one chunk per call, so they get a much lower cap.
|
||||
func requestBodyLimit(procedure string) int64 {
|
||||
if procedure == attachmentUploadProcedure {
|
||||
return attachmentUploadRequestLimit
|
||||
}
|
||||
return MaxAPIRequestBytes
|
||||
}
|
||||
|
||||
type APIV1Service struct {
|
||||
v1pb.UnimplementedInstanceServiceServer
|
||||
v1pb.UnimplementedAuthServiceServer
|
||||
|
|
@ -51,6 +60,7 @@ type APIV1Service struct {
|
|||
instanceStatsCache instanceStatsCache
|
||||
|
||||
linkMetadataFetcher linkMetadataFetcher
|
||||
attachmentUploads attachmentUploads
|
||||
}
|
||||
|
||||
// NewAPIV1Service creates an API v1 service with its shared dependencies.
|
||||
|
|
@ -132,6 +142,7 @@ func (s *APIV1Service) RegisterGateway(ctx context.Context, echoServer *echo.Ech
|
|||
if result != nil {
|
||||
r = r.WithContext(auth.ApplyToContext(ctx, result))
|
||||
}
|
||||
r.Body = http.MaxBytesReader(w, r.Body, requestBodyLimit(procedure))
|
||||
|
||||
next(w, r, pathParams)
|
||||
}
|
||||
|
|
@ -172,7 +183,7 @@ func (s *APIV1Service) RegisterGateway(ctx context.Context, echoServer *echo.Ech
|
|||
gwGroup := echoServer.Group("")
|
||||
// Register SSE endpoint with same CORS as rest of /api/v1.
|
||||
RegisterSSERoutes(gwGroup, s.SSEHub, s.Store, s.Secret)
|
||||
handler := echo.WrapHandler(http.MaxBytesHandler(gwMux, MaxAPIRequestBytes))
|
||||
handler := echo.WrapHandler(gwMux)
|
||||
|
||||
gwGroup.Any("/api/v1/*", handler)
|
||||
gwGroup.Any("/file/*", handler)
|
||||
|
|
@ -187,10 +198,17 @@ func (s *APIV1Service) RegisterGateway(ctx context.Context, echoServer *echo.Ech
|
|||
)
|
||||
connectMux := http.NewServeMux()
|
||||
connectHandler := NewConnectServiceHandler(s)
|
||||
connectHandler.RegisterConnectHandlers(connectMux, connectInterceptors, connect.WithReadMaxBytes(MaxAPIRequestBytes))
|
||||
connectHandler.RegisterConnectHandlers(connectMux, connectInterceptors,
|
||||
// Bound the decompressed message as well as the wire bytes below.
|
||||
connect.WithConditionalHandlerOptions(func(spec connect.Spec) []connect.HandlerOption {
|
||||
return []connect.HandlerOption{connect.WithReadMaxBytes(int(requestBodyLimit(spec.Procedure)))}
|
||||
}))
|
||||
|
||||
connectGroup := echoServer.Group("")
|
||||
connectGroup.Any("/memos.api.v1.*", echo.WrapHandler(http.MaxBytesHandler(connectMux, MaxAPIRequestBytes)))
|
||||
connectGroup.Any("/memos.api.v1.*", echo.WrapHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
r.Body = http.MaxBytesReader(w, r.Body, requestBodyLimit(r.URL.Path))
|
||||
connectMux.ServeHTTP(w, r)
|
||||
})))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,9 +30,9 @@ type Server struct {
|
|||
Profile *profile.Profile
|
||||
Store *store.Store
|
||||
|
||||
echoServer *echo.Echo
|
||||
httpServer *http.Server
|
||||
sseHub *apiv1.SSEHub
|
||||
echoServer *echo.Echo
|
||||
httpServer *http.Server
|
||||
apiV1Service *apiv1.APIV1Service
|
||||
}
|
||||
|
||||
func NewServer(ctx context.Context, profile *profile.Profile, store *store.Store) (*Server, error) {
|
||||
|
|
@ -66,7 +66,7 @@ func NewServer(ctx context.Context, profile *profile.Profile, store *store.Store
|
|||
frontend.NewFrontendService(profile, store).Serve(ctx, echoServer)
|
||||
|
||||
apiV1Service := apiv1.NewAPIV1Service(s.Secret, profile, store)
|
||||
s.sseHub = apiV1Service.SSEHub
|
||||
s.apiV1Service = apiV1Service
|
||||
|
||||
// Register HTTP file server routes BEFORE gRPC-Gateway to ensure proper range request handling for Safari.
|
||||
// This uses native HTTP serving (http.ServeContent) instead of gRPC for video/audio files.
|
||||
|
|
@ -127,6 +127,7 @@ func (s *Server) Shutdown(ctx context.Context) {
|
|||
|
||||
s.closeLongLivedConnections()
|
||||
s.shutdownHTTPServer(ctx)
|
||||
s.apiV1Service.CloseAttachmentUploads()
|
||||
|
||||
// Close database connection.
|
||||
if err := s.Store.Close(); err != nil {
|
||||
|
|
@ -138,9 +139,7 @@ func (s *Server) Shutdown(ctx context.Context) {
|
|||
|
||||
func (s *Server) closeLongLivedConnections() {
|
||||
// Long-lived SSE requests do not finish on their own during http.Server.Shutdown.
|
||||
if s.sseHub != nil {
|
||||
s.sseHub.Close()
|
||||
}
|
||||
s.apiV1Service.SSEHub.Close()
|
||||
}
|
||||
|
||||
func (s *Server) shutdownHTTPServer(ctx context.Context) {
|
||||
|
|
|
|||
|
|
@ -1,35 +1,74 @@
|
|||
import { create } from "@bufbuild/protobuf";
|
||||
import { Code, ConnectError } from "@connectrpc/connect";
|
||||
import { attachmentServiceClient } from "@/connect";
|
||||
import type { Attachment } from "@/types/proto/api/v1/attachment_service_pb";
|
||||
import { AttachmentSchema, MotionMediaSchema } from "@/types/proto/api/v1/attachment_service_pb";
|
||||
import type { Attachment, UploadAttachmentRequest } from "@/types/proto/api/v1/attachment_service_pb";
|
||||
import { AttachmentSchema, MotionMediaSchema, UploadAttachmentSpecSchema } from "@/types/proto/api/v1/attachment_service_pb";
|
||||
import type { LocalFile } from "../types/attachment";
|
||||
|
||||
const DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024;
|
||||
// The server allows eight unfinished uploads per user; stay well under it.
|
||||
const MAX_CONCURRENT_UPLOADS = 4;
|
||||
|
||||
export const uploadService = {
|
||||
async uploadFile(localFile: LocalFile): Promise<Attachment> {
|
||||
async uploadFile(localFile: LocalFile, signal?: AbortSignal): Promise<Attachment> {
|
||||
const { file, motionMedia } = localFile;
|
||||
const [mediaMetadata, arrayBuffer] = await Promise.all([localFile.mediaMetadata, file.arrayBuffer()]);
|
||||
const buffer = new Uint8Array(arrayBuffer);
|
||||
return attachmentServiceClient.createAttachment({
|
||||
const mediaMetadata = await localFile.mediaMetadata;
|
||||
const spec = create(UploadAttachmentSpecSchema, {
|
||||
attachment: create(AttachmentSchema, {
|
||||
filename: file.name,
|
||||
size: BigInt(file.size),
|
||||
type: file.type,
|
||||
content: buffer,
|
||||
motionMedia: motionMedia ? create(MotionMediaSchema, motionMedia) : undefined,
|
||||
mediaMetadata,
|
||||
}),
|
||||
totalSize: BigInt(file.size),
|
||||
});
|
||||
// The spec call carries no data so that retrying it after a lost response
|
||||
// can at worst orphan an upload, never create a duplicate attachment.
|
||||
const initial = await attachmentServiceClient.uploadAttachment({ upload: { case: "spec", value: spec } }, { signal });
|
||||
if (!initial.uploadId || initial.committedSize !== 0n || initial.maxChunkSize <= 0) {
|
||||
throw new Error("Invalid upload initialization response");
|
||||
}
|
||||
const chunkSize = Math.min(initial.maxChunkSize, DEFAULT_CHUNK_SIZE);
|
||||
for (let offset = 0; ; ) {
|
||||
signal?.throwIfAborted();
|
||||
const end = Math.min(offset + chunkSize, file.size);
|
||||
const request: Pick<UploadAttachmentRequest, "upload" | "writeOffset" | "data" | "finishWrite"> = {
|
||||
upload: { case: "uploadId", value: initial.uploadId },
|
||||
writeOffset: BigInt(offset),
|
||||
data: new Uint8Array(await file.slice(offset, end).arrayBuffer()),
|
||||
finishWrite: end === file.size,
|
||||
};
|
||||
// Retrying the identical last write is safe even if its response was lost
|
||||
// after the server finalized the attachment.
|
||||
let response;
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
try {
|
||||
response = await attachmentServiceClient.uploadAttachment(request, { signal });
|
||||
break;
|
||||
} catch (error) {
|
||||
if (signal?.aborted || ConnectError.from(error).code !== Code.Unavailable || attempt >= 2) throw error;
|
||||
}
|
||||
}
|
||||
if (response.committedSize !== BigInt(end)) {
|
||||
throw new Error("Unexpected upload offset");
|
||||
}
|
||||
if (request.finishWrite) {
|
||||
if (!response.attachment) throw new Error("Upload completed without an attachment");
|
||||
return response.attachment;
|
||||
}
|
||||
offset = end;
|
||||
}
|
||||
},
|
||||
|
||||
async uploadFiles(localFiles: LocalFile[]): Promise<Attachment[]> {
|
||||
if (localFiles.length === 0) return [];
|
||||
|
||||
const attachments: Attachment[] = [];
|
||||
|
||||
for (const localFile of localFiles) {
|
||||
attachments.push(await uploadService.uploadFile(localFile));
|
||||
}
|
||||
|
||||
const attachments: Attachment[] = new Array(localFiles.length);
|
||||
let next = 0;
|
||||
const worker = async () => {
|
||||
for (let index = next++; index < localFiles.length; index = next++) {
|
||||
attachments[index] = await this.uploadFile(localFiles[index]);
|
||||
}
|
||||
};
|
||||
await Promise.all(Array.from({ length: Math.min(MAX_CONCURRENT_UPLOADS, localFiles.length) }, worker));
|
||||
return attachments;
|
||||
},
|
||||
};
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -4,14 +4,14 @@ import type { LocalFile } from "@/components/MemoEditor/types/attachment";
|
|||
import { MediaMetadataSchema } from "@/types/proto/api/v1/attachment_service_pb";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
createAttachment: vi.fn(),
|
||||
uploadAttachment: vi.fn(),
|
||||
extractMetadata: vi.fn(),
|
||||
}));
|
||||
|
||||
// The ingest helper lives beside useFileUpload, whose module graph reaches
|
||||
// AuthContext and the query hooks; stub every client they name-import.
|
||||
vi.mock("@/connect", () => ({
|
||||
attachmentServiceClient: { createAttachment: mocks.createAttachment },
|
||||
attachmentServiceClient: { uploadAttachment: mocks.uploadAttachment },
|
||||
authServiceClient: {},
|
||||
userServiceClient: {},
|
||||
memoViewServiceClient: {},
|
||||
|
|
@ -58,14 +58,19 @@ describe("media metadata at file ingest", () => {
|
|||
|
||||
describe("uploadService media metadata", () => {
|
||||
beforeEach(() => {
|
||||
mocks.createAttachment.mockImplementation(async ({ attachment }) => attachment);
|
||||
mocks.uploadAttachment.mockImplementation(async ({ data }) => ({
|
||||
uploadId: "test-id",
|
||||
maxChunkSize: 2 * 1024 * 1024,
|
||||
committedSize: BigInt(data?.length ?? 0),
|
||||
attachment: data ? { name: "attachments/test" } : undefined,
|
||||
}));
|
||||
});
|
||||
|
||||
it("submits no metadata for files ingested without any", async () => {
|
||||
await uploadService.uploadFiles([localImage()]);
|
||||
|
||||
expect(mocks.createAttachment).toHaveBeenCalledOnce();
|
||||
expect(mocks.createAttachment.mock.calls[0][0].attachment.mediaMetadata).toBeUndefined();
|
||||
expect(mocks.uploadAttachment).toHaveBeenCalledTimes(2);
|
||||
expect(mocks.uploadAttachment.mock.calls[0][0].upload.value.attachment.mediaMetadata).toBeUndefined();
|
||||
});
|
||||
|
||||
it("submits the metadata extracted at ingest", async () => {
|
||||
|
|
@ -73,14 +78,14 @@ describe("uploadService media metadata", () => {
|
|||
|
||||
await uploadService.uploadFiles([localImage(Promise.resolve(metadata))]);
|
||||
|
||||
expect(mocks.createAttachment.mock.calls[0][0].attachment.mediaMetadata).toEqual(metadata);
|
||||
expect(mocks.uploadAttachment.mock.calls[0][0].upload.value.attachment.mediaMetadata).toEqual(metadata);
|
||||
});
|
||||
|
||||
it("continues without metadata when extraction produced no usable values", async () => {
|
||||
await uploadService.uploadFiles([localImage(Promise.resolve(undefined))]);
|
||||
|
||||
expect(mocks.createAttachment).toHaveBeenCalledOnce();
|
||||
expect(mocks.createAttachment.mock.calls[0][0].attachment.mediaMetadata).toBeUndefined();
|
||||
expect(mocks.uploadAttachment).toHaveBeenCalledTimes(2);
|
||||
expect(mocks.uploadAttachment.mock.calls[0][0].upload.value.attachment.mediaMetadata).toBeUndefined();
|
||||
});
|
||||
|
||||
it("submits ingest-time metadata end to end", async () => {
|
||||
|
|
@ -90,6 +95,6 @@ describe("uploadService media metadata", () => {
|
|||
|
||||
await uploadService.uploadFiles(toLocalFiles([file], { createBlobUrl, saveMediaMetadata: true }));
|
||||
|
||||
expect(mocks.createAttachment.mock.calls[0][0].attachment.mediaMetadata).toEqual(metadata);
|
||||
expect(mocks.uploadAttachment.mock.calls[0][0].upload.value.attachment.mediaMetadata).toEqual(metadata);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
98
web/tests/upload-service.test.ts
Normal file
98
web/tests/upload-service.test.ts
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
import { create } from "@bufbuild/protobuf";
|
||||
import { Code, ConnectError } from "@connectrpc/connect";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { AttachmentSchema } from "@/types/proto/api/v1/attachment_service_pb";
|
||||
|
||||
const { uploadAttachment } = vi.hoisted(() => ({ uploadAttachment: vi.fn() }));
|
||||
vi.mock("@/connect", () => ({ attachmentServiceClient: { uploadAttachment } }));
|
||||
|
||||
import { uploadService } from "@/components/MemoEditor/services/uploadService";
|
||||
|
||||
const attachment = create(AttachmentSchema, { name: "attachments/uploaded" });
|
||||
const localFile = (content: string) => ({
|
||||
file: new File([content], "test.txt", { type: "text/plain" }),
|
||||
origin: "upload" as const,
|
||||
previewUrl: "blob:test",
|
||||
});
|
||||
|
||||
describe("chunked attachment uploads", () => {
|
||||
beforeEach(() => {
|
||||
uploadAttachment.mockImplementation(async (request) => ({
|
||||
uploadId: "opaque-id",
|
||||
maxChunkSize: 3,
|
||||
committedSize: request.data ? request.writeOffset + BigInt(request.data.length) : 0n,
|
||||
attachment: request.finishWrite ? attachment : undefined,
|
||||
}));
|
||||
});
|
||||
|
||||
it("reads only file slices and writes consecutive offsets", async () => {
|
||||
const local = localFile("abcdefgh");
|
||||
const wholeFileRead = vi.spyOn(local.file, "arrayBuffer").mockRejectedValue(new Error("must not read the whole file"));
|
||||
const slice = vi.spyOn(local.file, "slice");
|
||||
|
||||
await expect(uploadService.uploadFile(local)).resolves.toEqual(attachment);
|
||||
|
||||
expect(wholeFileRead).not.toHaveBeenCalled();
|
||||
expect(slice.mock.calls).toEqual([
|
||||
[0, 3],
|
||||
[3, 6],
|
||||
[6, 8],
|
||||
]);
|
||||
const requests = uploadAttachment.mock.calls.map(([request]) => request);
|
||||
expect(requests[0].upload.case).toBe("spec");
|
||||
expect(requests[0].upload.value.totalSize).toBe(8n);
|
||||
expect(requests[0].upload.value.attachment.content).toHaveLength(0);
|
||||
expect(requests[0].data).toBeUndefined();
|
||||
expect(requests.slice(1).map((request) => request.upload)).toEqual(Array(3).fill({ case: "uploadId", value: "opaque-id" }));
|
||||
expect(requests.slice(1).map((request) => request.writeOffset)).toEqual([0n, 3n, 6n]);
|
||||
expect(requests.slice(1).map((request) => new TextDecoder().decode(request.data))).toEqual(["abc", "def", "gh"]);
|
||||
expect(requests.slice(1).map((request) => request.finishWrite)).toEqual([false, false, true]);
|
||||
});
|
||||
|
||||
it("finalizes an empty file", async () => {
|
||||
await expect(uploadService.uploadFile(localFile(""))).resolves.toEqual(attachment);
|
||||
expect(uploadAttachment).toHaveBeenCalledTimes(2);
|
||||
expect(uploadAttachment.mock.calls[1][0]).toMatchObject({ writeOffset: 0n, finishWrite: true });
|
||||
expect(uploadAttachment.mock.calls[1][0].data).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("retries an identical final chunk after a lost response", async () => {
|
||||
const original = uploadAttachment.getMockImplementation();
|
||||
let lost = false;
|
||||
uploadAttachment.mockImplementation(async (request) => {
|
||||
if (request.finishWrite && !lost) {
|
||||
lost = true;
|
||||
throw new ConnectError("response lost", Code.Unavailable);
|
||||
}
|
||||
return original?.(request);
|
||||
});
|
||||
|
||||
await expect(uploadService.uploadFile(localFile("abc"))).resolves.toEqual(attachment);
|
||||
expect(uploadAttachment.mock.calls[1][0]).toBe(uploadAttachment.mock.calls[2][0]);
|
||||
});
|
||||
|
||||
it("does not retry permission errors", async () => {
|
||||
uploadAttachment.mockRejectedValueOnce(new ConnectError("denied", Code.PermissionDenied));
|
||||
await expect(uploadService.uploadFile(localFile("abc"))).rejects.toMatchObject({ code: Code.PermissionDenied });
|
||||
expect(uploadAttachment).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("bounds retries and stops on cancellation", async () => {
|
||||
const initial = { uploadId: "id", committedSize: 0n, maxChunkSize: 3 };
|
||||
uploadAttachment.mockResolvedValueOnce(initial).mockRejectedValue(new ConnectError("offline", Code.Unavailable));
|
||||
await expect(uploadService.uploadFile(localFile("abc"))).rejects.toMatchObject({ code: Code.Unavailable });
|
||||
expect(uploadAttachment).toHaveBeenCalledTimes(4);
|
||||
|
||||
uploadAttachment.mockReset().mockResolvedValue(initial);
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
await expect(uploadService.uploadFile(localFile("abc"), controller.signal)).rejects.toThrow();
|
||||
expect(uploadAttachment).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("rejects invalid progress instead of looping or skipping bytes", async () => {
|
||||
uploadAttachment.mockResolvedValue({ uploadId: "id", committedSize: 0n, maxChunkSize: 3 });
|
||||
await expect(uploadService.uploadFile(localFile("abcdef"))).rejects.toThrow("Unexpected upload offset");
|
||||
expect(uploadAttachment).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
Loading…
Reference in a new issue