Add UploadAttachment, a unary RPC that accepts a file in bounded chunks and streams it to a temp file instead of holding the whole blob in one request body and in memory. A call carrying a spec starts an upload and returns an opaque upload_id; later calls carry that id with write_offset, data, and finish_write, following the Cloud Storage WriteObject shape. Retrying the last chunk after a lost response is idempotent, uploads are bound to their owner, and they expire after 30 minutes of inactivity. CreateAttachment and the chunked finalize now share one processing pipeline for motion-photo detection, EXIF stripping, and storage. The motion-photo detector works over an io.ReaderAt so large JPEGs are never loaded whole, and the request body cap is a single per-procedure lookup. The web editor uploads through the new RPC in 2 MiB chunks with bounded concurrency under the server's per-user limit.
98 lines
4.6 KiB
TypeScript
98 lines
4.6 KiB
TypeScript
import { create } from "@bufbuild/protobuf";
|
|
import { Code, ConnectError } from "@connectrpc/connect";
|
|
import { beforeEach, describe, expect, it, vi } from "vitest";
|
|
import { AttachmentSchema } from "@/types/proto/api/v1/attachment_service_pb";
|
|
|
|
const { uploadAttachment } = vi.hoisted(() => ({ uploadAttachment: vi.fn() }));
|
|
vi.mock("@/connect", () => ({ attachmentServiceClient: { uploadAttachment } }));
|
|
|
|
import { uploadService } from "@/components/MemoEditor/services/uploadService";
|
|
|
|
const attachment = create(AttachmentSchema, { name: "attachments/uploaded" });
|
|
const localFile = (content: string) => ({
|
|
file: new File([content], "test.txt", { type: "text/plain" }),
|
|
origin: "upload" as const,
|
|
previewUrl: "blob:test",
|
|
});
|
|
|
|
describe("chunked attachment uploads", () => {
|
|
beforeEach(() => {
|
|
uploadAttachment.mockImplementation(async (request) => ({
|
|
uploadId: "opaque-id",
|
|
maxChunkSize: 3,
|
|
committedSize: request.data ? request.writeOffset + BigInt(request.data.length) : 0n,
|
|
attachment: request.finishWrite ? attachment : undefined,
|
|
}));
|
|
});
|
|
|
|
it("reads only file slices and writes consecutive offsets", async () => {
|
|
const local = localFile("abcdefgh");
|
|
const wholeFileRead = vi.spyOn(local.file, "arrayBuffer").mockRejectedValue(new Error("must not read the whole file"));
|
|
const slice = vi.spyOn(local.file, "slice");
|
|
|
|
await expect(uploadService.uploadFile(local)).resolves.toEqual(attachment);
|
|
|
|
expect(wholeFileRead).not.toHaveBeenCalled();
|
|
expect(slice.mock.calls).toEqual([
|
|
[0, 3],
|
|
[3, 6],
|
|
[6, 8],
|
|
]);
|
|
const requests = uploadAttachment.mock.calls.map(([request]) => request);
|
|
expect(requests[0].upload.case).toBe("spec");
|
|
expect(requests[0].upload.value.totalSize).toBe(8n);
|
|
expect(requests[0].upload.value.attachment.content).toHaveLength(0);
|
|
expect(requests[0].data).toBeUndefined();
|
|
expect(requests.slice(1).map((request) => request.upload)).toEqual(Array(3).fill({ case: "uploadId", value: "opaque-id" }));
|
|
expect(requests.slice(1).map((request) => request.writeOffset)).toEqual([0n, 3n, 6n]);
|
|
expect(requests.slice(1).map((request) => new TextDecoder().decode(request.data))).toEqual(["abc", "def", "gh"]);
|
|
expect(requests.slice(1).map((request) => request.finishWrite)).toEqual([false, false, true]);
|
|
});
|
|
|
|
it("finalizes an empty file", async () => {
|
|
await expect(uploadService.uploadFile(localFile(""))).resolves.toEqual(attachment);
|
|
expect(uploadAttachment).toHaveBeenCalledTimes(2);
|
|
expect(uploadAttachment.mock.calls[1][0]).toMatchObject({ writeOffset: 0n, finishWrite: true });
|
|
expect(uploadAttachment.mock.calls[1][0].data).toHaveLength(0);
|
|
});
|
|
|
|
it("retries an identical final chunk after a lost response", async () => {
|
|
const original = uploadAttachment.getMockImplementation();
|
|
let lost = false;
|
|
uploadAttachment.mockImplementation(async (request) => {
|
|
if (request.finishWrite && !lost) {
|
|
lost = true;
|
|
throw new ConnectError("response lost", Code.Unavailable);
|
|
}
|
|
return original?.(request);
|
|
});
|
|
|
|
await expect(uploadService.uploadFile(localFile("abc"))).resolves.toEqual(attachment);
|
|
expect(uploadAttachment.mock.calls[1][0]).toBe(uploadAttachment.mock.calls[2][0]);
|
|
});
|
|
|
|
it("does not retry permission errors", async () => {
|
|
uploadAttachment.mockRejectedValueOnce(new ConnectError("denied", Code.PermissionDenied));
|
|
await expect(uploadService.uploadFile(localFile("abc"))).rejects.toMatchObject({ code: Code.PermissionDenied });
|
|
expect(uploadAttachment).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it("bounds retries and stops on cancellation", async () => {
|
|
const initial = { uploadId: "id", committedSize: 0n, maxChunkSize: 3 };
|
|
uploadAttachment.mockResolvedValueOnce(initial).mockRejectedValue(new ConnectError("offline", Code.Unavailable));
|
|
await expect(uploadService.uploadFile(localFile("abc"))).rejects.toMatchObject({ code: Code.Unavailable });
|
|
expect(uploadAttachment).toHaveBeenCalledTimes(4);
|
|
|
|
uploadAttachment.mockReset().mockResolvedValue(initial);
|
|
const controller = new AbortController();
|
|
controller.abort();
|
|
await expect(uploadService.uploadFile(localFile("abc"), controller.signal)).rejects.toThrow();
|
|
expect(uploadAttachment).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it("rejects invalid progress instead of looping or skipping bytes", async () => {
|
|
uploadAttachment.mockResolvedValue({ uploadId: "id", committedSize: 0n, maxChunkSize: 3 });
|
|
await expect(uploadService.uploadFile(localFile("abcdef"))).rejects.toThrow("Unexpected upload offset");
|
|
expect(uploadAttachment).toHaveBeenCalledTimes(2);
|
|
});
|
|
});
|