refactor(fileserver): fix stale README and simplify file serving (#6208)
This commit is contained in:
parent
16cd3107bf
commit
cf1be15f3e
4 changed files with 79 additions and 435 deletions
|
|
@ -26,8 +26,6 @@ const (
|
||||||
// This is unrelated to maximum upload size limit, which is now set through system setting.
|
// This is unrelated to maximum upload size limit, which is now set through system setting.
|
||||||
MaxUploadBufferSizeBytes = 32 << 20
|
MaxUploadBufferSizeBytes = 32 << 20
|
||||||
MebiByte = 1024 * 1024
|
MebiByte = 1024 * 1024
|
||||||
// ThumbnailCacheFolder is the folder name where the thumbnail images are stored.
|
|
||||||
ThumbnailCacheFolder = ".thumbnail_cache"
|
|
||||||
|
|
||||||
// defaultJPEGQuality is the JPEG quality used when re-encoding images for EXIF stripping.
|
// defaultJPEGQuality is the JPEG quality used when re-encoding images for EXIF stripping.
|
||||||
// Quality 95 maintains visual quality while ensuring metadata is removed.
|
// Quality 95 maintains visual quality while ensuring metadata is removed.
|
||||||
|
|
@ -36,11 +34,6 @@ const (
|
||||||
maxImagePixels = 50_000_000
|
maxImagePixels = 50_000_000
|
||||||
)
|
)
|
||||||
|
|
||||||
var SupportedThumbnailMimeTypes = []string{
|
|
||||||
"image/png",
|
|
||||||
"image/jpeg",
|
|
||||||
}
|
|
||||||
|
|
||||||
// exifCapableImageTypes defines image formats that may contain EXIF metadata.
|
// exifCapableImageTypes defines image formats that may contain EXIF metadata.
|
||||||
// These formats will have their EXIF metadata stripped on upload for privacy.
|
// These formats will have their EXIF metadata stripped on upload for privacy.
|
||||||
var exifCapableImageTypes = map[string]bool{
|
var exifCapableImageTypes = map[string]bool{
|
||||||
|
|
|
||||||
|
|
@ -1,304 +1,46 @@
|
||||||
# Fileserver Package
|
# Fileserver Package
|
||||||
|
|
||||||
## Overview
|
The `fileserver` package serves binary content (attachments, avatars) over plain HTTP instead of gRPC, so that HTTP range requests work — required for Safari video/audio playback ([RFC 9110 §14](https://www.rfc-editor.org/rfc/rfc9110#section-14)). Metadata stays on the gRPC API; only bytes are served here.
|
||||||
|
|
||||||
The `fileserver` package handles all binary file serving for Memos using native HTTP handlers. It was created to replace gRPC-based binary serving, which had limitations with HTTP range requests (required for Safari video/audio playback).
|
## Endpoints
|
||||||
|
|
||||||
## Responsibilities
|
|
||||||
|
|
||||||
- Serve attachment binary files (images, videos, audio, documents)
|
|
||||||
- Serve user avatar images
|
|
||||||
- Handle HTTP range requests for video/audio streaming
|
|
||||||
- Authenticate requests using JWT tokens or Personal Access Tokens
|
|
||||||
- Check permissions for private content
|
|
||||||
- Generate and serve image thumbnails
|
|
||||||
- Prevent XSS attacks on uploaded content
|
|
||||||
- Support S3 external storage
|
|
||||||
|
|
||||||
## Architecture
|
|
||||||
|
|
||||||
### Design Principles
|
|
||||||
|
|
||||||
1. **Separation of Concerns**: Binary files via HTTP, metadata via gRPC
|
|
||||||
2. **DRY**: Imports auth constants from `api/v1` package (single source of truth)
|
|
||||||
3. **Security First**: Authentication, authorization, and XSS prevention
|
|
||||||
4. **Performance**: Native HTTP streaming with proper caching headers
|
|
||||||
|
|
||||||
### Package Structure
|
|
||||||
|
|
||||||
|
```text
|
||||||
|
GET /file/attachments/:uid[/:filename] # attachment binary
|
||||||
|
?thumbnail=true # JPEG thumbnail for supported image types
|
||||||
|
?motion=true # embedded motion-photo video clip
|
||||||
|
?share_token={uid} # access via a memo share link
|
||||||
|
GET /file/users/:identifier/avatar # user avatar (by username)
|
||||||
```
|
```
|
||||||
fileserver/
|
|
||||||
├── fileserver.go # Main service and HTTP handlers
|
|
||||||
├── README.md # This file
|
|
||||||
└── fileserver_test.go # Tests (to be added)
|
|
||||||
```
|
|
||||||
|
|
||||||
## API Endpoints
|
|
||||||
|
|
||||||
### 1. Attachment Binary
|
|
||||||
```
|
|
||||||
GET /file/attachments/:uid/:filename[?thumbnail=true]
|
|
||||||
```
|
|
||||||
|
|
||||||
**Parameters:**
|
|
||||||
- `uid` - Attachment unique identifier
|
|
||||||
- `filename` - Original filename
|
|
||||||
- `thumbnail` (optional) - Return thumbnail for images
|
|
||||||
|
|
||||||
**Authentication:** Required for non-public memos
|
|
||||||
|
|
||||||
**Response:**
|
|
||||||
- `200 OK` - File content with proper Content-Type
|
|
||||||
- `206 Partial Content` - For range requests (video/audio)
|
|
||||||
- `401 Unauthorized` - Authentication required
|
|
||||||
- `403 Forbidden` - User not authorized
|
|
||||||
- `404 Not Found` - Attachment not found
|
|
||||||
|
|
||||||
**Headers:**
|
|
||||||
- `Content-Type` - MIME type of the file
|
|
||||||
- `Cache-Control: public, max-age=3600`
|
|
||||||
- `Accept-Ranges: bytes` - For video/audio
|
|
||||||
- `Content-Range` - For partial responses (206)
|
|
||||||
|
|
||||||
### 2. User Avatar
|
|
||||||
```
|
|
||||||
GET /file/users/:identifier/avatar
|
|
||||||
```
|
|
||||||
|
|
||||||
**Parameters:**
|
|
||||||
- `identifier` - User ID (e.g., `1`) or username (e.g., `steven`)
|
|
||||||
|
|
||||||
**Authentication:** Not required (avatars are public)
|
|
||||||
|
|
||||||
**Response:**
|
|
||||||
- `200 OK` - Avatar image (PNG/JPEG)
|
|
||||||
- `404 Not Found` - User not found or no avatar set
|
|
||||||
|
|
||||||
**Headers:**
|
|
||||||
- `Content-Type` - image/png or image/jpeg
|
|
||||||
- `Cache-Control: public, max-age=3600`
|
|
||||||
|
|
||||||
## Authentication
|
## Authentication
|
||||||
|
|
||||||
### Supported Methods
|
Handled by `server/auth` (see [authenticator.go](../../auth/authenticator.go) and [token.go](../../auth/token.go) for the constants and token formats). Priority: `Authorization: Bearer` header (access token or personal access token) first, then the refresh token cookie. See `getCurrentUser` in [fileserver.go](fileserver.go).
|
||||||
|
|
||||||
The fileserver supports the following authentication methods:
|
## Authorization
|
||||||
|
|
||||||
1. **JWT Access Token** (`Authorization: Bearer {token}`)
|
Attachment access follows memo visibility, evaluated by `server/access.CheckMemoRead`:
|
||||||
- Short-lived tokens (15 minutes) for API access
|
|
||||||
- Stateless validation using JWT signature
|
|
||||||
- Extracts user ID from token claims
|
|
||||||
|
|
||||||
2. **Personal Access Token (PAT)** (`Authorization: Bearer {pat}`)
|
- Public memo: no auth required (when the instance allows anonymous access)
|
||||||
- Long-lived tokens for programmatic access
|
- Protected memo: any authenticated user
|
||||||
- Validates against database for revocation
|
- Private memo: creator only
|
||||||
- Prefixed with specific identifier
|
- Valid `share_token`: grants access to that memo's attachments
|
||||||
|
- Unlinked attachment (no memo): creator or admin only
|
||||||
|
|
||||||
### Authentication Flow
|
Avatars are public on instances that allow anonymous access; private instances require authentication.
|
||||||
|
|
||||||
```
|
## Serving behavior
|
||||||
Request → getCurrentUser()
|
|
||||||
├─→ Try Session Cookie
|
|
||||||
│ ├─→ Parse cookie value
|
|
||||||
│ ├─→ Get user from DB
|
|
||||||
│ ├─→ Validate session
|
|
||||||
│ └─→ Return user (if valid)
|
|
||||||
│
|
|
||||||
└─→ Try JWT Token
|
|
||||||
├─→ Parse Authorization header
|
|
||||||
├─→ Verify JWT signature
|
|
||||||
├─→ Get user from DB
|
|
||||||
├─→ Validate token in access tokens list
|
|
||||||
└─→ Return user (if valid)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Permission Model
|
- **Video/audio** are streamed with range-request support (`http.ServeFile` / `http.ServeContent`); S3-backed media redirects to a presigned URL.
|
||||||
|
- **Thumbnails** are generated at max 600px, cached in `{data_dir}/.thumbnail_cache/`, with a semaphore capping concurrent generation. Images with HDR/wide-gamut metadata are served as originals, since re-encoding would strip it.
|
||||||
**Attachments:**
|
- **Motion photos** have their embedded video extracted and cached in `{data_dir}/.motion_cache/`.
|
||||||
- Unlinked: Public (no auth required)
|
- **XSS prevention**: script-capable MIME types are rewritten to `application/octet-stream`, non-media files get `Content-Disposition: attachment`, and all responses carry `X-Content-Type-Options: nosniff` plus a restrictive `Content-Security-Policy`.
|
||||||
- Public memo: Public (no auth required)
|
- **Caching**: public attachments get `public, no-cache`; private ones `private, no-store`; avatars and thumbnails `public, max-age=3600`.
|
||||||
- Protected memo: Requires authentication
|
|
||||||
- Private memo: Creator only
|
|
||||||
|
|
||||||
**Avatars:**
|
|
||||||
- Always public (no auth required)
|
|
||||||
|
|
||||||
## Key Functions
|
|
||||||
|
|
||||||
### HTTP Handlers
|
|
||||||
|
|
||||||
#### `serveAttachmentFile(c echo.Context) error`
|
|
||||||
Main handler for attachment binary serving.
|
|
||||||
|
|
||||||
**Flow:**
|
|
||||||
1. Extract UID from URL parameter
|
|
||||||
2. Fetch attachment from database
|
|
||||||
3. Check permissions (memo visibility)
|
|
||||||
4. Get binary blob (local file, S3, or database)
|
|
||||||
5. Handle thumbnail request (if applicable)
|
|
||||||
6. Set security headers (XSS prevention)
|
|
||||||
7. Serve with range request support (video/audio)
|
|
||||||
|
|
||||||
#### `serveUserAvatar(c echo.Context) error`
|
|
||||||
Main handler for user avatar serving.
|
|
||||||
|
|
||||||
**Flow:**
|
|
||||||
1. Extract identifier (ID or username) from URL
|
|
||||||
2. Lookup user in database
|
|
||||||
3. Check if avatar exists
|
|
||||||
4. Decode base64 data URI
|
|
||||||
5. Serve with proper content type and caching
|
|
||||||
|
|
||||||
### Authentication
|
|
||||||
|
|
||||||
#### `getCurrentUser(ctx, c) (*store.User, error)`
|
|
||||||
Authenticates request using session cookie or JWT token.
|
|
||||||
|
|
||||||
#### `authenticateBySession(ctx, cookie) (*store.User, error)`
|
|
||||||
Validates session cookie and returns authenticated user.
|
|
||||||
|
|
||||||
#### `authenticateByJWT(ctx, token) (*store.User, error)`
|
|
||||||
Validates JWT access token and returns authenticated user.
|
|
||||||
|
|
||||||
### Permission Checks
|
|
||||||
|
|
||||||
#### `checkAttachmentPermission(ctx, c, attachment) error`
|
|
||||||
Validates user has permission to access attachment based on memo visibility.
|
|
||||||
|
|
||||||
### File Operations
|
|
||||||
|
|
||||||
#### `getAttachmentBlob(attachment) ([]byte, error)`
|
|
||||||
Retrieves binary content from local storage, S3, or database.
|
|
||||||
|
|
||||||
#### `getOrGenerateThumbnail(ctx, attachment) ([]byte, error)`
|
|
||||||
Returns cached thumbnail or generates new one (with semaphore limiting).
|
|
||||||
|
|
||||||
### Utilities
|
|
||||||
|
|
||||||
#### `getUserByIdentifier(ctx, identifier) (*store.User, error)`
|
|
||||||
Finds user by ID (int) or username (string).
|
|
||||||
|
|
||||||
#### `extractImageInfo(dataURI) (type, base64, error)`
|
|
||||||
Parses data URI to extract MIME type and base64 data.
|
|
||||||
|
|
||||||
## Dependencies
|
|
||||||
|
|
||||||
### External Packages
|
|
||||||
- `github.com/labstack/echo/v5` - HTTP router and middleware
|
|
||||||
- `github.com/golang-jwt/jwt/v5` - JWT parsing and validation
|
|
||||||
- `github.com/disintegration/imaging` - Image thumbnail generation
|
|
||||||
- `golang.org/x/sync/semaphore` - Concurrency control for thumbnails
|
|
||||||
|
|
||||||
### Internal Packages
|
|
||||||
- `server/auth` - Authentication utilities
|
|
||||||
- `store` - Database operations
|
|
||||||
- `internal/profile` - Server configuration
|
|
||||||
- `internal/storage/s3` - S3 storage client
|
|
||||||
|
|
||||||
## Configuration
|
|
||||||
|
|
||||||
### Constants
|
|
||||||
|
|
||||||
Auth-related constants are imported from `server/auth`:
|
|
||||||
- `auth.RefreshTokenCookieName` - "memos_refresh"
|
|
||||||
- `auth.PersonalAccessTokenPrefix` - PAT identifier prefix
|
|
||||||
|
|
||||||
Package-specific constants:
|
|
||||||
- `ThumbnailCacheFolder` - ".thumbnail_cache"
|
|
||||||
- `thumbnailMaxSize` - 600px
|
|
||||||
- `SupportedThumbnailMimeTypes` - ["image/png", "image/jpeg"]
|
|
||||||
|
|
||||||
## Error Handling
|
|
||||||
|
|
||||||
All handlers return Echo HTTP errors with appropriate status codes:
|
|
||||||
|
|
||||||
```go
|
|
||||||
// Bad request
|
|
||||||
echo.NewHTTPError(http.StatusBadRequest, "message")
|
|
||||||
|
|
||||||
// Unauthorized (no auth)
|
|
||||||
echo.NewHTTPError(http.StatusUnauthorized, "message")
|
|
||||||
|
|
||||||
// Forbidden (auth but no permission)
|
|
||||||
echo.NewHTTPError(http.StatusForbidden, "message")
|
|
||||||
|
|
||||||
// Not found
|
|
||||||
echo.NewHTTPError(http.StatusNotFound, "message")
|
|
||||||
|
|
||||||
// Internal error
|
|
||||||
echo.NewHTTPError(http.StatusInternalServerError, "message").SetInternal(err)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Security Considerations
|
|
||||||
|
|
||||||
### 1. XSS Prevention
|
|
||||||
SVG and HTML files are served as `application/octet-stream` to prevent script execution:
|
|
||||||
|
|
||||||
```go
|
|
||||||
if contentType == "image/svg+xml" ||
|
|
||||||
contentType == "text/html" ||
|
|
||||||
contentType == "application/xhtml+xml" {
|
|
||||||
contentType = "application/octet-stream"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. Authentication
|
|
||||||
Private content requires valid JWT access token or Personal Access Token.
|
|
||||||
|
|
||||||
### 3. Authorization
|
|
||||||
Memo visibility rules enforced before serving attachments.
|
|
||||||
|
|
||||||
### 4. Input Validation
|
|
||||||
- Attachment UID validated from database
|
|
||||||
- User identifier validated (ID or username)
|
|
||||||
- Range requests validated before processing
|
|
||||||
|
|
||||||
## Performance Optimizations
|
|
||||||
|
|
||||||
### 1. Thumbnail Caching
|
|
||||||
Thumbnails cached on disk to avoid regeneration:
|
|
||||||
- Cache location: `{data_dir}/.thumbnail_cache/`
|
|
||||||
- Filename: `{attachment_id}{extension}`
|
|
||||||
- Semaphore limits concurrent generation (max 3)
|
|
||||||
|
|
||||||
### 2. HTTP Range Requests
|
|
||||||
Video/audio files use `http.ServeContent()` for efficient streaming:
|
|
||||||
- Automatic range parsing
|
|
||||||
- Efficient memory usage (streaming, not loading full file)
|
|
||||||
- Safari-compatible partial content responses
|
|
||||||
|
|
||||||
### 3. Caching Headers
|
|
||||||
All responses include cache headers:
|
|
||||||
```
|
|
||||||
Cache-Control: public, max-age=3600
|
|
||||||
```
|
|
||||||
|
|
||||||
### 4. S3 External Links
|
|
||||||
S3 files served via presigned URLs (no server download).
|
|
||||||
|
|
||||||
## Testing
|
## Testing
|
||||||
|
|
||||||
### Unit Tests (To Add)
|
Unit tests live in [fileserver_test.go](fileserver_test.go), covering permission checks, streaming, thumbnails, and metadata detection. Manual checks:
|
||||||
See SAFARI_FIX.md for recommended test coverage.
|
|
||||||
|
|
||||||
### Manual Testing
|
|
||||||
```bash
|
```bash
|
||||||
# Test attachment
|
|
||||||
curl "http://localhost:8081/file/attachments/{uid}/file.jpg"
|
curl "http://localhost:8081/file/attachments/{uid}/file.jpg"
|
||||||
|
|
||||||
# Test avatar by username
|
|
||||||
curl "http://localhost:8081/file/users/steven/avatar"
|
|
||||||
|
|
||||||
# Test range request
|
|
||||||
curl -H "Range: bytes=0-999" "http://localhost:8081/file/attachments/{uid}/video.mp4"
|
curl -H "Range: bytes=0-999" "http://localhost:8081/file/attachments/{uid}/video.mp4"
|
||||||
```
|
```
|
||||||
|
|
||||||
## Future Improvements
|
|
||||||
|
|
||||||
See SAFARI_FIX.md section "Future Improvements" for planned enhancements.
|
|
||||||
|
|
||||||
## Related Documentation
|
|
||||||
|
|
||||||
- [SAFARI_FIX.md](../../../SAFARI_FIX.md) - Full migration guide
|
|
||||||
- [server/router/api/v1/auth.go](../api/v1/auth.go) - Auth constants source of truth
|
|
||||||
- [RFC 7233](https://tools.ietf.org/html/rfc7233) - HTTP Range Requests spec
|
|
||||||
|
|
|
||||||
|
|
@ -29,11 +29,11 @@ import (
|
||||||
|
|
||||||
// Constants for file serving configuration.
|
// Constants for file serving configuration.
|
||||||
const (
|
const (
|
||||||
// ThumbnailCacheFolder is the folder name where thumbnail images are stored.
|
// thumbnailCacheFolder is the folder name where thumbnail images are stored.
|
||||||
ThumbnailCacheFolder = ".thumbnail_cache"
|
thumbnailCacheFolder = ".thumbnail_cache"
|
||||||
|
|
||||||
// MotionCacheFolder is the folder name where extracted motion clips are stored.
|
// motionCacheFolder is the folder name where extracted motion clips are stored.
|
||||||
MotionCacheFolder = ".motion_cache"
|
motionCacheFolder = ".motion_cache"
|
||||||
|
|
||||||
// thumbnailMaxSize is the maximum dimension (width or height) for thumbnails.
|
// thumbnailMaxSize is the maximum dimension (width or height) for thumbnails.
|
||||||
thumbnailMaxSize = 600
|
thumbnailMaxSize = 600
|
||||||
|
|
@ -85,16 +85,6 @@ var avatarAllowedTypes = map[string]bool{
|
||||||
"image/heif": true,
|
"image/heif": true,
|
||||||
}
|
}
|
||||||
|
|
||||||
// SupportedThumbnailMimeTypes is the exported list of thumbnail-supported MIME types.
|
|
||||||
var SupportedThumbnailMimeTypes = []string{
|
|
||||||
"image/png",
|
|
||||||
"image/jpeg",
|
|
||||||
"image/jpg",
|
|
||||||
"image/heic",
|
|
||||||
"image/heif",
|
|
||||||
"image/webp",
|
|
||||||
}
|
|
||||||
|
|
||||||
var errUseOriginalForThumbnail = errors.New("serve original image instead of metadata-stripping thumbnail")
|
var errUseOriginalForThumbnail = errors.New("serve original image instead of metadata-stripping thumbnail")
|
||||||
|
|
||||||
// dataURIRegex parses data URI format: data:image/png;base64,iVBORw0KGgo...
|
// dataURIRegex parses data URI format: data:image/png;base64,iVBORw0KGgo...
|
||||||
|
|
@ -163,7 +153,7 @@ func (s *FileServerService) serveAttachmentFile(c *echo.Context) error {
|
||||||
return s.serveMotionClip(c, attachment)
|
return s.serveMotionClip(c, attachment)
|
||||||
}
|
}
|
||||||
|
|
||||||
contentType := s.sanitizeContentType(attachment.Type)
|
contentType := sanitizeContentType(attachment.Type)
|
||||||
|
|
||||||
// Stream video/audio to avoid loading entire file into memory.
|
// Stream video/audio to avoid loading entire file into memory.
|
||||||
if isMediaType(attachment.Type) {
|
if isMediaType(attachment.Type) {
|
||||||
|
|
@ -191,7 +181,7 @@ func (s *FileServerService) serveUserAvatar(c *echo.Context) error {
|
||||||
|
|
||||||
identifier := c.Param("identifier")
|
identifier := c.Param("identifier")
|
||||||
|
|
||||||
user, err := s.getUserByUsername(ctx, identifier)
|
user, err := s.Store.GetUser(ctx, &store.FindUser{Username: &identifier})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return echo.NewHTTPError(http.StatusInternalServerError, "failed to get user").Wrap(err)
|
return echo.NewHTTPError(http.StatusInternalServerError, "failed to get user").Wrap(err)
|
||||||
}
|
}
|
||||||
|
|
@ -202,7 +192,7 @@ func (s *FileServerService) serveUserAvatar(c *echo.Context) error {
|
||||||
return echo.NewHTTPError(http.StatusNotFound, "avatar not found")
|
return echo.NewHTTPError(http.StatusNotFound, "avatar not found")
|
||||||
}
|
}
|
||||||
|
|
||||||
imageType, imageData, err := s.parseDataURI(user.AvatarURL)
|
imageType, imageData, err := parseDataURI(user.AvatarURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return echo.NewHTTPError(http.StatusInternalServerError, "failed to parse avatar data").Wrap(err)
|
return echo.NewHTTPError(http.StatusInternalServerError, "failed to parse avatar data").Wrap(err)
|
||||||
}
|
}
|
||||||
|
|
@ -212,7 +202,6 @@ func (s *FileServerService) serveUserAvatar(c *echo.Context) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
setSecurityHeaders(c)
|
setSecurityHeaders(c)
|
||||||
c.Response().Header().Set(echo.HeaderContentType, imageType)
|
|
||||||
c.Response().Header().Set(echo.HeaderCacheControl, cacheMaxAge)
|
c.Response().Header().Set(echo.HeaderCacheControl, cacheMaxAge)
|
||||||
|
|
||||||
return c.Blob(http.StatusOK, imageType, imageData)
|
return c.Blob(http.StatusOK, imageType, imageData)
|
||||||
|
|
@ -229,11 +218,7 @@ func (s *FileServerService) serveMediaStream(c *echo.Context, attachment *store.
|
||||||
|
|
||||||
switch attachment.StorageType {
|
switch attachment.StorageType {
|
||||||
case storepb.AttachmentStorageType_LOCAL:
|
case storepb.AttachmentStorageType_LOCAL:
|
||||||
filePath, err := s.resolveLocalPath(attachment.Reference)
|
http.ServeFile(c.Response(), c.Request(), s.resolveLocalPath(attachment.Reference))
|
||||||
if err != nil {
|
|
||||||
return echo.NewHTTPError(http.StatusInternalServerError, "failed to resolve file path").Wrap(err)
|
|
||||||
}
|
|
||||||
http.ServeFile(c.Response(), c.Request(), filePath)
|
|
||||||
return nil
|
return nil
|
||||||
|
|
||||||
case storepb.AttachmentStorageType_S3:
|
case storepb.AttachmentStorageType_S3:
|
||||||
|
|
@ -276,11 +261,7 @@ func (s *FileServerService) serveStaticFile(c *echo.Context, attachment *store.A
|
||||||
|
|
||||||
switch attachment.StorageType {
|
switch attachment.StorageType {
|
||||||
case storepb.AttachmentStorageType_LOCAL:
|
case storepb.AttachmentStorageType_LOCAL:
|
||||||
filePath, err := s.resolveLocalPath(attachment.Reference)
|
http.ServeFile(c.Response(), c.Request(), s.resolveLocalPath(attachment.Reference))
|
||||||
if err != nil {
|
|
||||||
return echo.NewHTTPError(http.StatusInternalServerError, "failed to resolve file path").Wrap(err)
|
|
||||||
}
|
|
||||||
http.ServeFile(c.Response(), c.Request(), filePath)
|
|
||||||
return nil
|
return nil
|
||||||
case storepb.AttachmentStorageType_S3:
|
case storepb.AttachmentStorageType_S3:
|
||||||
reader, err := s.getAttachmentReader(c.Request().Context(), attachment)
|
reader, err := s.getAttachmentReader(c.Request().Context(), attachment)
|
||||||
|
|
@ -299,28 +280,25 @@ func (s *FileServerService) serveStaticFile(c *echo.Context, attachment *store.A
|
||||||
// =============================================================================
|
// =============================================================================
|
||||||
|
|
||||||
// getAttachmentBlob retrieves the binary content of an attachment from storage.
|
// getAttachmentBlob retrieves the binary content of an attachment from storage.
|
||||||
func (s *FileServerService) getAttachmentBlob(attachment *store.Attachment) ([]byte, error) {
|
func (s *FileServerService) getAttachmentBlob(ctx context.Context, attachment *store.Attachment) ([]byte, error) {
|
||||||
switch attachment.StorageType {
|
reader, err := s.getAttachmentReader(ctx, attachment)
|
||||||
case storepb.AttachmentStorageType_LOCAL:
|
if err != nil {
|
||||||
return s.readLocalFile(attachment.Reference)
|
return nil, err
|
||||||
|
|
||||||
case storepb.AttachmentStorageType_S3:
|
|
||||||
return s.downloadFromS3(context.Background(), attachment)
|
|
||||||
|
|
||||||
default:
|
|
||||||
return attachment.Blob, nil
|
|
||||||
}
|
}
|
||||||
|
defer reader.Close()
|
||||||
|
|
||||||
|
blob, err := io.ReadAll(reader)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "failed to read attachment content")
|
||||||
|
}
|
||||||
|
return blob, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// getAttachmentReader returns a reader for streaming attachment content.
|
// getAttachmentReader returns a reader for streaming attachment content.
|
||||||
func (s *FileServerService) getAttachmentReader(ctx context.Context, attachment *store.Attachment) (io.ReadCloser, error) {
|
func (s *FileServerService) getAttachmentReader(ctx context.Context, attachment *store.Attachment) (io.ReadCloser, error) {
|
||||||
switch attachment.StorageType {
|
switch attachment.StorageType {
|
||||||
case storepb.AttachmentStorageType_LOCAL:
|
case storepb.AttachmentStorageType_LOCAL:
|
||||||
filePath, err := s.resolveLocalPath(attachment.Reference)
|
file, err := os.Open(s.resolveLocalPath(attachment.Reference))
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
file, err := os.Open(filePath)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
return nil, errors.Wrap(err, "file not found")
|
return nil, errors.Wrap(err, "file not found")
|
||||||
|
|
@ -346,49 +324,12 @@ func (s *FileServerService) getAttachmentReader(ctx context.Context, attachment
|
||||||
}
|
}
|
||||||
|
|
||||||
// resolveLocalPath converts a storage reference to an absolute file path.
|
// resolveLocalPath converts a storage reference to an absolute file path.
|
||||||
func (s *FileServerService) resolveLocalPath(reference string) (string, error) {
|
func (s *FileServerService) resolveLocalPath(reference string) string {
|
||||||
filePath := filepath.FromSlash(reference)
|
filePath := filepath.FromSlash(reference)
|
||||||
if !filepath.IsAbs(filePath) {
|
if !filepath.IsAbs(filePath) {
|
||||||
filePath = filepath.Join(s.Profile.Data, filePath)
|
filePath = filepath.Join(s.Profile.Data, filePath)
|
||||||
}
|
}
|
||||||
return filePath, nil
|
return filePath
|
||||||
}
|
|
||||||
|
|
||||||
// readLocalFile reads the entire contents of a local file.
|
|
||||||
func (s *FileServerService) readLocalFile(reference string) ([]byte, error) {
|
|
||||||
filePath, err := s.resolveLocalPath(reference)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
file, err := os.Open(filePath)
|
|
||||||
if err != nil {
|
|
||||||
if os.IsNotExist(err) {
|
|
||||||
return nil, errors.Wrap(err, "file not found")
|
|
||||||
}
|
|
||||||
return nil, errors.Wrap(err, "failed to open file")
|
|
||||||
}
|
|
||||||
defer file.Close()
|
|
||||||
|
|
||||||
blob, err := io.ReadAll(file)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "failed to read file")
|
|
||||||
}
|
|
||||||
return blob, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// downloadFromS3 downloads the entire object from S3.
|
|
||||||
func (s *FileServerService) downloadFromS3(ctx context.Context, attachment *store.Attachment) ([]byte, error) {
|
|
||||||
driver, s3Object, err := s.Store.ResolveAttachmentS3Driver(ctx, attachment)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
blob, err := driver.GetObject(ctx, s3Object.Key)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "failed to download from S3")
|
|
||||||
}
|
|
||||||
return blob, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// getS3PresignedURL generates a presigned URL for direct S3 access.
|
// getS3PresignedURL generates a presigned URL for direct S3 access.
|
||||||
|
|
@ -418,7 +359,7 @@ func (s *FileServerService) getOrGenerateThumbnail(ctx context.Context, attachme
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fast path: return cached thumbnail if exists.
|
// Fast path: return cached thumbnail if exists.
|
||||||
if blob, err := s.readCachedThumbnail(thumbnailPath); err == nil {
|
if blob, err := os.ReadFile(thumbnailPath); err == nil {
|
||||||
return blob, nil
|
return blob, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -437,7 +378,7 @@ func (s *FileServerService) getOrGenerateThumbnail(ctx context.Context, attachme
|
||||||
defer s.thumbnailSemaphore.Release(1)
|
defer s.thumbnailSemaphore.Release(1)
|
||||||
|
|
||||||
// Double-check after acquiring semaphore (another goroutine may have generated it).
|
// Double-check after acquiring semaphore (another goroutine may have generated it).
|
||||||
if blob, err := s.readCachedThumbnail(thumbnailPath); err == nil {
|
if blob, err := os.ReadFile(thumbnailPath); err == nil {
|
||||||
return blob, nil
|
return blob, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -446,7 +387,7 @@ func (s *FileServerService) getOrGenerateThumbnail(ctx context.Context, attachme
|
||||||
|
|
||||||
// getThumbnailPath returns the file path for a cached thumbnail.
|
// getThumbnailPath returns the file path for a cached thumbnail.
|
||||||
func (s *FileServerService) getThumbnailPath(attachment *store.Attachment) (string, error) {
|
func (s *FileServerService) getThumbnailPath(attachment *store.Attachment) (string, error) {
|
||||||
cacheFolder := filepath.Join(s.Profile.Data, ThumbnailCacheFolder)
|
cacheFolder := filepath.Join(s.Profile.Data, thumbnailCacheFolder)
|
||||||
if err := os.MkdirAll(cacheFolder, os.ModePerm); err != nil {
|
if err := os.MkdirAll(cacheFolder, os.ModePerm); err != nil {
|
||||||
return "", errors.Wrap(err, "failed to create thumbnail cache folder")
|
return "", errors.Wrap(err, "failed to create thumbnail cache folder")
|
||||||
}
|
}
|
||||||
|
|
@ -474,14 +415,10 @@ func (s *FileServerService) shouldUseOriginalForThumbnail(ctx context.Context, a
|
||||||
return false, errors.Wrap(err, "failed to read image metadata probe")
|
return false, errors.Wrap(err, "failed to read image metadata probe")
|
||||||
}
|
}
|
||||||
|
|
||||||
return hasThumbnailSensitiveMetadata(attachment.Type, probe), nil
|
return hasThumbnailSensitiveMetadata(probe), nil
|
||||||
}
|
|
||||||
|
|
||||||
func hasThumbnailSensitiveMetadata(mimeType string, data []byte) bool {
|
|
||||||
if mimeType == "image/heic" || mimeType == "image/heif" {
|
|
||||||
return true
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func hasThumbnailSensitiveMetadata(data []byte) bool {
|
||||||
for _, marker := range [][]byte{
|
for _, marker := range [][]byte{
|
||||||
[]byte("ICC_PROFILE"),
|
[]byte("ICC_PROFILE"),
|
||||||
[]byte("iCCP"),
|
[]byte("iCCP"),
|
||||||
|
|
@ -520,16 +457,6 @@ func hasThumbnailSensitiveMetadata(mimeType string, data []byte) bool {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// readCachedThumbnail reads a thumbnail from the cache directory.
|
|
||||||
func (*FileServerService) readCachedThumbnail(path string) ([]byte, error) {
|
|
||||||
file, err := os.Open(path)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer file.Close()
|
|
||||||
return io.ReadAll(file)
|
|
||||||
}
|
|
||||||
|
|
||||||
// generateThumbnail creates a new thumbnail and saves it to disk.
|
// generateThumbnail creates a new thumbnail and saves it to disk.
|
||||||
func (s *FileServerService) generateThumbnail(ctx context.Context, attachment *store.Attachment, thumbnailPath string) ([]byte, error) {
|
func (s *FileServerService) generateThumbnail(ctx context.Context, attachment *store.Attachment, thumbnailPath string) ([]byte, error) {
|
||||||
reader, err := s.getAttachmentReader(ctx, attachment)
|
reader, err := s.getAttachmentReader(ctx, attachment)
|
||||||
|
|
@ -548,17 +475,15 @@ func (s *FileServerService) generateThumbnail(ctx context.Context, attachment *s
|
||||||
|
|
||||||
thumbnailImage := imaging.Resize(img, thumbnailWidth, thumbnailHeight, imaging.Lanczos)
|
thumbnailImage := imaging.Resize(img, thumbnailWidth, thumbnailHeight, imaging.Lanczos)
|
||||||
|
|
||||||
output, err := os.Create(thumbnailPath)
|
var buf bytes.Buffer
|
||||||
if err != nil {
|
if err := imaging.Encode(&buf, thumbnailImage, imaging.JPEG, imaging.JPEGQuality(90)); err != nil {
|
||||||
return nil, errors.Wrap(err, "failed to create thumbnail file")
|
return nil, errors.Wrap(err, "failed to encode thumbnail")
|
||||||
}
|
}
|
||||||
defer output.Close()
|
if err := os.WriteFile(thumbnailPath, buf.Bytes(), 0644); err != nil {
|
||||||
|
|
||||||
if err := imaging.Encode(output, thumbnailImage, imaging.JPEG, imaging.JPEGQuality(90)); err != nil {
|
|
||||||
return nil, errors.Wrap(err, "failed to save thumbnail")
|
return nil, errors.Wrap(err, "failed to save thumbnail")
|
||||||
}
|
}
|
||||||
|
|
||||||
return s.readCachedThumbnail(thumbnailPath)
|
return buf.Bytes(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// calculateThumbnailDimensions calculates the target dimensions for a thumbnail.
|
// calculateThumbnailDimensions calculates the target dimensions for a thumbnail.
|
||||||
|
|
@ -592,17 +517,17 @@ func (s *FileServerService) serveMotionClip(c *echo.Context, attachment *store.A
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *FileServerService) getOrExtractMotionClip(_ context.Context, attachment *store.Attachment) ([]byte, error) {
|
func (s *FileServerService) getOrExtractMotionClip(ctx context.Context, attachment *store.Attachment) ([]byte, error) {
|
||||||
motionPath, err := s.getMotionPath(attachment)
|
motionPath, err := s.getMotionPath(attachment)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if blob, err := s.readCachedThumbnail(motionPath); err == nil {
|
if blob, err := os.ReadFile(motionPath); err == nil {
|
||||||
return blob, nil
|
return blob, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
blob, err := s.getAttachmentBlob(attachment)
|
blob, err := s.getAttachmentBlob(ctx, attachment)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
@ -620,7 +545,7 @@ func (s *FileServerService) getOrExtractMotionClip(_ context.Context, attachment
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *FileServerService) getMotionPath(attachment *store.Attachment) (string, error) {
|
func (s *FileServerService) getMotionPath(attachment *store.Attachment) (string, error) {
|
||||||
cacheFolder := filepath.Join(s.Profile.Data, MotionCacheFolder)
|
cacheFolder := filepath.Join(s.Profile.Data, motionCacheFolder)
|
||||||
if err := os.MkdirAll(cacheFolder, os.ModePerm); err != nil {
|
if err := os.MkdirAll(cacheFolder, os.ModePerm); err != nil {
|
||||||
return "", errors.Wrap(err, "failed to create motion cache folder")
|
return "", errors.Wrap(err, "failed to create motion cache folder")
|
||||||
}
|
}
|
||||||
|
|
@ -712,17 +637,12 @@ func (s *FileServerService) getCurrentUser(ctx context.Context, c *echo.Context)
|
||||||
return s.authenticator.AuthenticateToUser(ctx, authHeader, cookieHeader)
|
return s.authenticator.AuthenticateToUser(ctx, authHeader, cookieHeader)
|
||||||
}
|
}
|
||||||
|
|
||||||
// getUserByUsername finds a user by username only.
|
|
||||||
func (s *FileServerService) getUserByUsername(ctx context.Context, identifier string) (*store.User, error) {
|
|
||||||
return s.Store.GetUser(ctx, &store.FindUser{Username: &identifier})
|
|
||||||
}
|
|
||||||
|
|
||||||
// =============================================================================
|
// =============================================================================
|
||||||
// Helper Functions
|
// Helper Functions
|
||||||
// =============================================================================
|
// =============================================================================
|
||||||
|
|
||||||
// sanitizeContentType converts potentially dangerous MIME types to safe alternatives.
|
// sanitizeContentType converts potentially dangerous MIME types to safe alternatives.
|
||||||
func (*FileServerService) sanitizeContentType(mimeType string) string {
|
func sanitizeContentType(mimeType string) string {
|
||||||
contentType := mimeType
|
contentType := mimeType
|
||||||
if strings.HasPrefix(contentType, "text/") {
|
if strings.HasPrefix(contentType, "text/") {
|
||||||
contentType += "; charset=utf-8"
|
contentType += "; charset=utf-8"
|
||||||
|
|
@ -735,7 +655,7 @@ func (*FileServerService) sanitizeContentType(mimeType string) string {
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseDataURI extracts MIME type and decoded data from a data URI.
|
// parseDataURI extracts MIME type and decoded data from a data URI.
|
||||||
func (*FileServerService) parseDataURI(dataURI string) (string, []byte, error) {
|
func parseDataURI(dataURI string) (string, []byte, error) {
|
||||||
matches := dataURIRegex.FindStringSubmatch(dataURI)
|
matches := dataURIRegex.FindStringSubmatch(dataURI)
|
||||||
if len(matches) != 3 {
|
if len(matches) != 3 {
|
||||||
return "", nil, errors.New("invalid data URI format")
|
return "", nil, errors.New("invalid data URI format")
|
||||||
|
|
|
||||||
|
|
@ -598,37 +598,26 @@ func TestServeAttachmentFile_ThumbnailWithSensitiveMetadataServesOriginal(t *tes
|
||||||
func TestHasThumbnailSensitiveMetadata(t *testing.T) {
|
func TestHasThumbnailSensitiveMetadata(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
mimeType string
|
|
||||||
data []byte
|
data []byte
|
||||||
want bool
|
want bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "jpeg hdr gain map",
|
name: "jpeg hdr gain map",
|
||||||
mimeType: "image/jpeg",
|
|
||||||
data: []byte("xmp hdrgm:Version=\"1.0\""),
|
data: []byte("xmp hdrgm:Version=\"1.0\""),
|
||||||
want: true,
|
want: true,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "jpeg icc profile",
|
name: "jpeg icc profile",
|
||||||
mimeType: "image/jpeg",
|
|
||||||
data: []byte("ICC_PROFILE"),
|
data: []byte("ICC_PROFILE"),
|
||||||
want: true,
|
want: true,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "png cicp chunk",
|
name: "png cicp chunk",
|
||||||
mimeType: "image/png",
|
|
||||||
data: []byte("cICP"),
|
data: []byte("cICP"),
|
||||||
want: true,
|
want: true,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
name: "heic",
|
|
||||||
mimeType: "image/heic",
|
|
||||||
data: nil,
|
|
||||||
want: true,
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
name: "plain jpeg",
|
name: "plain jpeg",
|
||||||
mimeType: "image/jpeg",
|
|
||||||
data: []byte("plain image data"),
|
data: []byte("plain image data"),
|
||||||
want: false,
|
want: false,
|
||||||
},
|
},
|
||||||
|
|
@ -636,7 +625,7 @@ func TestHasThumbnailSensitiveMetadata(t *testing.T) {
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
require.Equal(t, tt.want, hasThumbnailSensitiveMetadata(tt.mimeType, tt.data))
|
require.Equal(t, tt.want, hasThumbnailSensitiveMetadata(tt.data))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue