#R&D #A\u200dB
\n", html) -} - -func TestRenderHTMLRejectsUnclosedReferenceDestination(t *testing.T) { - svc := NewService(WithTagExtension()) - html, err := svc.RenderHTML([]byte("[#use][bad]\n\n[bad]:(")) - require.NoError(t, err) - assert.Equal(t, "[#use][bad]
\n[bad]:(
\n", html) -} - -func TestRenderHTMLRecognizesGFMEmails(t *testing.T) { - svc := NewService(WithTagExtension()) - tests := []struct { - content string - expected string - }{ - { - content: "foo#mail@example.com", - expected: "foo#mail@example.com
\n", - }, - { - content: "#foo/bar_baz@example.com", - expected: "#foo/bar_baz@example.com
\n", - }, - { - content: "_foo@example.com #tag_", - expected: "foo@example.com #tag
\n", - }, - { - content: "foo@bar.com@baz.example", - expected: "foo@bar.com@baz.example
\n", - }, - { - content: "foo@bar.com+abc@def.com", - expected: "\n", - }, - { - content: `#foo\+bar@example.com`, - expected: "\n", - }, - { - content: `#foo\@example\.com`, - expected: "\n", - }, - { - content: `#foo.bar@example.com`, - expected: "\n", - }, - { - content: `#foo.bar@example.com`, - expected: "\n", - }, - { - content: `#foo@example.com`, - expected: "\n", - }, - } - for _, test := range tests { - html, err := svc.RenderHTML([]byte(test.content)) - require.NoError(t, err) - assert.Equal(t, test.expected, html) - } -} - -func TestRenderHTMLRecognizesGFMEmailWithoutTagExtension(t *testing.T) { - svc := NewService() - html, err := svc.RenderHTML([]byte("mail@example.com")) - require.NoError(t, err) - assert.Equal(t, "\n", html) -} - -func TestRenderHTMLRecognizesGFMURLsInsideUnresolvedBracketText(t *testing.T) { +func TestExtractAllRecognizesGFMURLsInsideUnresolvedBracketText(t *testing.T) { svc := NewService(WithTagExtension()) for _, content := range []string{ "[ https://example.com/#hidden]", @@ -928,10 +849,6 @@ func TestRenderHTMLRecognizesGFMURLsInsideUnresolvedBracketText(t *testing.T) { "![ https://example.com/#hidden]", "[text https://example.com/#hidden", } { - html, err := svc.RenderHTML([]byte(content)) - require.NoError(t, err) - assert.Contains(t, html, "$x < y$\n$$meta\nx < y\n$$", html) } func TestRenderMarkdownPreservesLineBreakAfterTag(t *testing.T) { diff --git a/server/router/frontend/frontend.go b/server/router/frontend/frontend.go index e8769f5f..45a14592 100644 --- a/server/router/frontend/frontend.go +++ b/server/router/frontend/frontend.go @@ -80,7 +80,7 @@ func spaFallbackMiddleware(frontendFS fs.FS) echo.MiddlewareFunc { } func shouldSkipFrontendStatic(requestPath string) bool { - if requestPath == "/robots.txt" || requestPath == "/sitemap.xml" || strings.HasSuffix(requestPath, "/rss.xml") { + if requestPath == "/robots.txt" || requestPath == "/sitemap.xml" { return true } return hasPathPrefix(requestPath, "/api") || diff --git a/server/router/mcp/README.md b/server/router/mcp/README.md index 63bbe086..d7fe91f9 100644 --- a/server/router/mcp/README.md +++ b/server/router/mcp/README.md @@ -15,7 +15,7 @@ source of truth and reuses the API's authentication and authorization as-is. ## Integration -`server.NewServer` calls `mcp.NewMCPService` after registering the API, file, RSS, and gRPC-gateway routes, passing the same Echo server: +`server.NewServer` calls `mcp.NewMCPService` after registering the API, file, and gRPC-gateway routes, passing the same Echo server: ```go mcpService, err := mcp.NewMCPService(profile, echoServer) diff --git a/server/router/rss/rss.go b/server/router/rss/rss.go deleted file mode 100644 index 3784cd1f..00000000 --- a/server/router/rss/rss.go +++ /dev/null @@ -1,365 +0,0 @@ -package rss - -import ( - "context" - "crypto/sha256" - "fmt" - "net/http" - "regexp" - "strconv" - "strings" - "time" - - "github.com/gorilla/feeds" - "github.com/labstack/echo/v5" - - "github.com/usememos/memos/internal/markdown" - storepb "github.com/usememos/memos/proto/gen/store" - "github.com/usememos/memos/store" -) - -const ( - maxRSSItemCount = 100 -) - -var ( - // Regex to match markdown headings at the start of a line. - markdownHeadingRegex = regexp.MustCompile(`^#{1,6}\s*`) -) - -type RSSService struct { - Store *store.Store - MarkdownService markdown.Service -} - -type RSSHeading struct { - Title string - Description string - Language string -} - -// NewRSSService creates an RSS service backed by the store and markdown renderer. -func NewRSSService(store *store.Store, markdownService markdown.Service) *RSSService { - return &RSSService{ - Store: store, - MarkdownService: markdownService, - } -} - -func (s *RSSService) RegisterRoutes(g *echo.Group) { - g.GET("/explore/rss.xml", s.GetExploreRSS) - g.GET("/u/:username/rss.xml", s.GetUserRSS) -} - -func (s *RSSService) GetExploreRSS(c *echo.Context) error { - ctx := c.Request().Context() - allowAnonymous, err := s.Store.AllowsAnonymousAccess(ctx) - if err != nil { - return echo.NewHTTPError(http.StatusInternalServerError, "failed to get instance access policy").Wrap(err) - } - if !allowAnonymous { - return echo.NewHTTPError(http.StatusNotFound, "RSS is unavailable") - } - - normalStatus := store.Normal - limit := maxRSSItemCount - memoFind := store.FindMemo{ - RowStatus: &normalStatus, - VisibilityList: []store.Visibility{store.Public}, - Access: &store.MemoAccessScope{AllowPublic: true}, - ExcludeComments: true, - Limit: &limit, - } - memoList, err := s.Store.ListMemos(ctx, &memoFind) - if err != nil { - return echo.NewHTTPError(http.StatusInternalServerError, "Failed to find memo list").Wrap(err) - } - - baseURL := c.Scheme() + "://" + c.Request().Host - rss, lastModified, err := s.generateRSSFromMemoList(ctx, memoList, baseURL, nil) - if err != nil { - return echo.NewHTTPError(http.StatusInternalServerError, "Failed to generate rss").Wrap(err) - } - - etag := rssETag(rss) - s.setRSSHeaders(c, etag, lastModified) - if c.Request().Header.Get("If-None-Match") == etag { - return c.NoContent(http.StatusNotModified) - } - return c.String(http.StatusOK, rss) -} - -func (s *RSSService) GetUserRSS(c *echo.Context) error { - ctx := c.Request().Context() - allowAnonymous, err := s.Store.AllowsAnonymousAccess(ctx) - if err != nil { - return echo.NewHTTPError(http.StatusInternalServerError, "failed to get instance access policy").Wrap(err) - } - if !allowAnonymous { - return echo.NewHTTPError(http.StatusNotFound, "RSS is unavailable") - } - - username := c.Param("username") - user, err := s.Store.GetUser(ctx, &store.FindUser{ - Username: &username, - }) - if err != nil { - return echo.NewHTTPError(http.StatusInternalServerError, "Failed to find user").Wrap(err) - } - if user == nil { - return echo.NewHTTPError(http.StatusNotFound, "User not found") - } - - normalStatus := store.Normal - limit := maxRSSItemCount - memoFind := store.FindMemo{ - CreatorID: &user.ID, - RowStatus: &normalStatus, - VisibilityList: []store.Visibility{store.Public}, - Access: &store.MemoAccessScope{AllowPublic: true}, - ExcludeComments: true, - Limit: &limit, - } - memoList, err := s.Store.ListMemos(ctx, &memoFind) - if err != nil { - return echo.NewHTTPError(http.StatusInternalServerError, "Failed to find memo list").Wrap(err) - } - - baseURL := c.Scheme() + "://" + c.Request().Host - rss, lastModified, err := s.generateRSSFromMemoList(ctx, memoList, baseURL, user) - if err != nil { - return echo.NewHTTPError(http.StatusInternalServerError, "Failed to generate rss").Wrap(err) - } - - etag := rssETag(rss) - s.setRSSHeaders(c, etag, lastModified) - if c.Request().Header.Get("If-None-Match") == etag { - return c.NoContent(http.StatusNotModified) - } - return c.String(http.StatusOK, rss) -} - -func (s *RSSService) generateRSSFromMemoList(ctx context.Context, memoList []*store.Memo, baseURL string, user *store.User) (string, time.Time, error) { - rssHeading, err := getRSSHeading(ctx, s.Store) - if err != nil { - return "", time.Time{}, err - } - - itemCountLimit := min(len(memoList), maxRSSItemCount) - // Derive channel timestamps only from feed data. Using the request time here - // changes an otherwise identical body and defeats ETag revalidation. - var lastModified time.Time - for _, memo := range memoList[:itemCountLimit] { - if updated := time.Unix(memo.UpdatedTs, 0); updated.After(lastModified) { - lastModified = updated - } - } - feed := &feeds.Feed{ - Title: rssHeading.Title, - Link: &feeds.Link{Href: baseURL}, - Description: rssHeading.Description, - Created: lastModified, - Updated: lastModified, - } - - if itemCountLimit == 0 { - // Return empty feed if no memos - rss, err := feed.ToRss() - return rss, time.Time{}, err - } - - // Batch load all attachments for all memos to avoid N+1 query problem - memoIDs := make([]int32, itemCountLimit) - for i := 0; i < itemCountLimit; i++ { - memoIDs[i] = memoList[i].ID - } - - allAttachments, err := s.Store.ListAttachments(ctx, &store.FindAttachment{ - MemoIDList: memoIDs, - }) - if err != nil { - return "", lastModified, err - } - - // Group attachments by memo ID for quick lookup - attachmentsByMemoID := make(map[int32][]*store.Attachment) - for _, attachment := range allAttachments { - if attachment.MemoID != nil { - attachmentsByMemoID[*attachment.MemoID] = append(attachmentsByMemoID[*attachment.MemoID], attachment) - } - } - - // Batch load all memo creators - creatorMap := make(map[int32]*store.User) - if user != nil { - // Single user feed - reuse the user object - creatorMap[user.ID] = user - } else { - // Multi-user feed - batch load all unique creators - creatorIDList := []int32{} - creatorIDMap := make(map[int32]bool) - for _, memo := range memoList[:itemCountLimit] { - if !creatorIDMap[memo.CreatorID] { - creatorIDList = append(creatorIDList, memo.CreatorID) - creatorIDMap[memo.CreatorID] = true - } - } - - // Batch load all users with a single query - users, err := s.Store.ListUsers(ctx, &store.FindUser{ - IDList: creatorIDList, - }) - if err != nil { - return "", lastModified, err - } - for _, creator := range users { - creatorMap[creator.ID] = creator - } - } - - // Generate feed items - feed.Items = make([]*feeds.Item, itemCountLimit) - for i := 0; i < itemCountLimit; i++ { - memo := memoList[i] - - // Generate item title from memo content - title := s.generateItemTitle(memo.Content) - - // Render content as HTML - htmlContent, err := s.getRSSItemDescription(memo.Content) - if err != nil { - return "", lastModified, err - } - - link := &feeds.Link{Href: baseURL + "/memos/" + memo.UID} - - item := &feeds.Item{ - Title: title, - Link: link, - Description: htmlContent, // Summary/excerpt - Content: htmlContent, // Full content in content:encoded - Created: time.Unix(memo.CreatedTs, 0), - Updated: time.Unix(memo.UpdatedTs, 0), - Id: link.Href, - } - - // Add author information - if creator, ok := creatorMap[memo.CreatorID]; ok { - authorName := creator.Nickname - if authorName == "" { - authorName = creator.Username - } - item.Author = &feeds.Author{ - Name: authorName, - Email: creator.Email, - } - } - - // Note: gorilla/feeds doesn't support categories in RSS items - // Tags could be added to the description or content if needed - - // Add first attachment as enclosure - if attachments, ok := attachmentsByMemoID[memo.ID]; ok && len(attachments) > 0 { - attachment := attachments[0] - enclosure := feeds.Enclosure{} - if attachment.StorageType == storepb.AttachmentStorageType_EXTERNAL { - enclosure.Url = attachment.Reference - } else { - enclosure.Url = fmt.Sprintf("%s/file/attachments/%s", baseURL, attachment.UID) - } - enclosure.Length = strconv.Itoa(int(attachment.Size)) - enclosure.Type = attachment.Type - item.Enclosure = &enclosure - } - - feed.Items[i] = item - } - - rss, err := feed.ToRss() - if err != nil { - return "", lastModified, err - } - return rss, lastModified, nil -} - -func (*RSSService) generateItemTitle(content string) string { - // Extract first line as title - lines := strings.Split(content, "\n") - title := strings.TrimSpace(lines[0]) - - // Remove markdown heading syntax using regex (handles # to ###### with optional spaces) - title = markdownHeadingRegex.ReplaceAllString(title, "") - title = strings.TrimSpace(title) - - // Limit title length - const maxTitleLength = 100 - if len(title) > maxTitleLength { - // Find last space before limit to avoid cutting words - cutoff := maxTitleLength - for i := min(maxTitleLength-1, len(title)-1); i > 0; i-- { - if title[i] == ' ' { - cutoff = i - break - } - } - if cutoff < maxTitleLength { - title = title[:cutoff] + "..." - } else { - // No space found, just truncate - title = title[:maxTitleLength] + "..." - } - } - - // If title is empty, use a default - if title == "" { - title = "Memo" - } - - return title -} - -func (s *RSSService) getRSSItemDescription(content string) (string, error) { - html, err := s.MarkdownService.RenderHTML([]byte(content)) - if err != nil { - return "", err - } - return html, nil -} - -func rssETag(content string) string { - hash := sha256.Sum256([]byte(content)) - return fmt.Sprintf(`"%x"`, hash[:8]) -} - -// setRSSHeaders sets appropriate HTTP headers for RSS responses. -func (*RSSService) setRSSHeaders(c *echo.Context, etag string, lastModified time.Time) { - c.Response().Header().Set(echo.HeaderContentType, "application/rss+xml; charset=utf-8") - // Revalidation is mandatory because an audience change must not leave a - // previously PUBLIC memo readable from a stale feed body. - c.Response().Header().Set(echo.HeaderCacheControl, "public, no-cache") - c.Response().Header().Set("ETag", etag) - if !lastModified.IsZero() { - c.Response().Header().Set("Last-Modified", lastModified.UTC().Format(http.TimeFormat)) - } -} - -func getRSSHeading(ctx context.Context, stores *store.Store) (RSSHeading, error) { - settings, err := stores.GetInstanceGeneralSetting(ctx) - if err != nil { - return RSSHeading{}, err - } - if settings == nil || settings.CustomProfile == nil { - return RSSHeading{ - Title: "Memos", - Description: "An open source, lightweight note-taking service. Easily capture and share your great thoughts.", - Language: "en-us", - }, nil - } - customProfile := settings.CustomProfile - - return RSSHeading{ - Title: customProfile.Title, - Description: customProfile.Description, - Language: "en-us", - }, nil -} diff --git a/server/router/rss/rss_test.go b/server/router/rss/rss_test.go deleted file mode 100644 index 887990f5..00000000 --- a/server/router/rss/rss_test.go +++ /dev/null @@ -1,180 +0,0 @@ -package rss - -import ( - "context" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" - - "github.com/labstack/echo/v5" - "github.com/stretchr/testify/require" - - "github.com/usememos/memos/internal/markdown" - storepb "github.com/usememos/memos/proto/gen/store" - "github.com/usememos/memos/store" - teststore "github.com/usememos/memos/store/test" -) - -func TestPublicRSSExcludesComments(t *testing.T) { - ctx := context.Background() - stores := teststore.NewTestingStore(ctx, t) - defer stores.Close() - setInstanceAccessMode(ctx, t, stores, storepb.InstanceAccessMode_INSTANCE_ACCESS_MODE_PUBLIC) - - user, err := stores.CreateUser(ctx, &store.User{ - Username: "rss-comment-owner", - Role: store.RoleUser, - Email: "rss-comment-owner@example.com", - }) - require.NoError(t, err) - - parent, err := stores.CreateMemo(ctx, &store.Memo{ - UID: "rss-public-parent", - CreatorID: user.ID, - Content: "public parent should stay in rss", - Visibility: store.Public, - }) - require.NoError(t, err) - - _, err = stores.CreateMemoComment(ctx, &store.Memo{ - UID: "rss-public-comment", - CreatorID: user.ID, - Content: "public comment should not be in rss", - Visibility: store.Private, - }, parent.ID, user.ID) - require.NoError(t, err) - - service := NewRSSService(stores, markdown.NewService()) - - exploreRSS := renderRSS(t, service, "/explore/rss.xml", "") - require.Contains(t, exploreRSS, "public parent should stay in rss") - require.NotContains(t, exploreRSS, "public comment should not be in rss") - - userRSS := renderRSS(t, service, "/u/rss-comment-owner/rss.xml", user.Username) - require.Contains(t, userRSS, "public parent should stay in rss") - require.NotContains(t, userRSS, "public comment should not be in rss") -} - -func TestPrivateInstanceDisablesRSS(t *testing.T) { - ctx := context.Background() - stores := teststore.NewTestingStore(ctx, t) - defer stores.Close() - setInstanceAccessMode(ctx, t, stores, storepb.InstanceAccessMode_INSTANCE_ACCESS_MODE_PRIVATE) - service := NewRSSService(stores, nil) - - for _, test := range []struct { - name string - target string - username string - }{ - {name: "explore", target: "/explore/rss.xml"}, - {name: "user", target: "/u/alice/rss.xml", username: "alice"}, - } { - t.Run(test.name, func(t *testing.T) { - e := echo.New() - req := httptest.NewRequest(http.MethodGet, test.target, strings.NewReader("")) - rec := httptest.NewRecorder() - c := e.NewContext(req, rec) - if test.username != "" { - c.SetPathValues(echo.PathValues{{Name: "username", Value: test.username}}) - } - - var err error - if test.username == "" { - err = service.GetExploreRSS(c) - } else { - err = service.GetUserRSS(c) - } - - var httpError *echo.HTTPError - require.ErrorAs(t, err, &httpError) - require.Equal(t, http.StatusNotFound, httpError.Code) - }) - } -} - -func TestRSSIfNoneMatchRemainsStableAcrossSeconds(t *testing.T) { - ctx := context.Background() - stores := teststore.NewTestingStore(ctx, t) - defer stores.Close() - setInstanceAccessMode(ctx, t, stores, storepb.InstanceAccessMode_INSTANCE_ACCESS_MODE_PUBLIC) - - user, err := stores.CreateUser(ctx, &store.User{ - Username: "rss-etag-owner", - Role: store.RoleUser, - Email: "rss-etag-owner@example.com", - }) - require.NoError(t, err) - _, err = stores.CreateMemo(ctx, &store.Memo{ - UID: "rss-etag-stable", - CreatorID: user.ID, - Content: "stable feed content", - Visibility: store.Public, - }) - require.NoError(t, err) - - service := NewRSSService(stores, markdown.NewService()) - explore := requestRSS(t, service, "/explore/rss.xml", "", "") - userFeed := requestRSS(t, service, "/u/rss-etag-owner/rss.xml", user.Username, "") - require.Equal(t, http.StatusOK, explore.Code) - require.Equal(t, http.StatusOK, userFeed.Code) - exploreETag := explore.Header().Get("ETag") - userETag := userFeed.Header().Get("ETag") - require.NotEmpty(t, exploreETag) - require.NotEmpty(t, userETag) - - // The previous request-time channel timestamp changed once per second even - // when no feed data changed, causing these conditional reads to return 200. - time.Sleep(1100 * time.Millisecond) - explore = requestRSS(t, service, "/explore/rss.xml", "", exploreETag) - userFeed = requestRSS(t, service, "/u/rss-etag-owner/rss.xml", user.Username, userETag) - require.Equal(t, http.StatusNotModified, explore.Code) - require.Equal(t, http.StatusNotModified, userFeed.Code) - require.Equal(t, exploreETag, explore.Header().Get("ETag")) - require.Equal(t, userETag, userFeed.Header().Get("ETag")) -} - -func setInstanceAccessMode(ctx context.Context, t *testing.T, stores *store.Store, mode storepb.InstanceAccessMode) { - t.Helper() - _, err := stores.UpsertInstanceSetting(ctx, &storepb.InstanceSetting{ - Key: storepb.InstanceSettingKey_ACCESS, - Value: &storepb.InstanceSetting_AccessSetting{AccessSetting: &storepb.InstanceAccessSetting{ - AccessMode: mode, - }}, - }) - require.NoError(t, err) -} - -func renderRSS(t *testing.T, service *RSSService, target string, username string) string { - t.Helper() - rec := requestRSS(t, service, target, username, "") - require.Equal(t, http.StatusOK, rec.Code) - return rec.Body.String() -} - -func requestRSS(t *testing.T, service *RSSService, target, username, ifNoneMatch string) *httptest.ResponseRecorder { - t.Helper() - - e := echo.New() - req := httptest.NewRequest(http.MethodGet, target, strings.NewReader("")) - req.Host = "example.com" - if ifNoneMatch != "" { - req.Header.Set("If-None-Match", ifNoneMatch) - } - rec := httptest.NewRecorder() - c := e.NewContext(req, rec) - if username != "" { - c.SetPathValues(echo.PathValues{{Name: "username", Value: username}}) - } - - var err error - if username == "" { - err = service.GetExploreRSS(c) - } else { - err = service.GetUserRSS(c) - } - require.NoError(t, err) - return rec -} diff --git a/server/server.go b/server/server.go index 0210344a..7c81279d 100644 --- a/server/server.go +++ b/server/server.go @@ -20,7 +20,6 @@ import ( "github.com/usememos/memos/server/router/fileserver" "github.com/usememos/memos/server/router/frontend" "github.com/usememos/memos/server/router/mcp" - "github.com/usememos/memos/server/router/rss" "github.com/usememos/memos/store" ) @@ -66,8 +65,6 @@ func NewServer(ctx context.Context, profile *profile.Profile, store *store.Store // Serve frontend static files. frontend.NewFrontendService(profile, store).Serve(ctx, echoServer) - rootGroup := echoServer.Group("") - apiV1Service := apiv1.NewAPIV1Service(s.Secret, profile, store) s.sseHub = apiV1Service.SSEHub @@ -76,9 +73,6 @@ func NewServer(ctx context.Context, profile *profile.Profile, store *store.Store fileServerService := fileserver.NewFileServerService(s.Profile, s.Store, s.Secret) fileServerService.RegisterRoutes(echoServer) - // Create and register RSS routes (needs markdown service from apiV1Service). - rss.NewRSSService(s.Store, apiV1Service.MarkdownService).RegisterRoutes(rootGroup) - // Register gRPC gateway as api v1 (includes SSE endpoint on CORS-enabled group). if err := apiV1Service.RegisterGateway(ctx, echoServer); err != nil { return nil, errors.Wrap(err, "failed to register gRPC gateway") diff --git a/server/test/startup_test.go b/server/test/startup_test.go index 40ed4940..d9af1fe3 100644 --- a/server/test/startup_test.go +++ b/server/test/startup_test.go @@ -2,7 +2,7 @@ // // Every other server test constructs apiv1.APIV1Service directly, which skips // server.NewServer entirely. That leaves route registration, gRPC-gateway -// wiring, MCP/RSS/fileserver/frontend mounting, CORS and the secret bootstrap +// wiring, MCP/fileserver/frontend mounting, CORS and the secret bootstrap // covered only by the Docker release script. These tests boot the real server // the same way cmd/memos/main.go does so a wiring regression fails in CI. package test @@ -289,12 +289,6 @@ func TestStartupServesEveryRegisteredRouter(t *testing.T) { require.Equal(t, http.StatusOK, resp.StatusCode, "public GET fallback should permit anonymous form posts") }) - t.Run("rss", func(t *testing.T) { - status, body := inst.do(t, http.MethodGet, "/explore/rss.xml", "", nil) - require.Equal(t, http.StatusOK, status, "rss route should be mounted: %s", body) - require.Contains(t, string(body), "