package store import ( "context" "io" "log/slog" "net/url" "os" "path/filepath" "regexp" "slices" "strings" "github.com/pkg/errors" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" "github.com/usememos/memos/internal/base" storepb "github.com/usememos/memos/proto/gen/store" ) const ( // DefaultDeploymentConfigurationDir is the directory scanned for runtime configuration files. DefaultDeploymentConfigurationDir = "/etc/secrets" maxDeploymentConfigurationSize = 1 << 20 maxTranscriptionModelLength = 256 maxTranscriptionLanguageLength = 32 maxTranscriptionPromptLength = 4096 ) var ( idpDeploymentFilenameMatcher = regexp.MustCompile(`^memos-idp-[a-z0-9]+(?:-[a-z0-9]+)*\.json$`) instanceSettingDeploymentFilenameMatcher = regexp.MustCompile(`^memos-instance-setting-[a-z0-9]+(?:-[a-z0-9]+)*\.json$`) protoJSONUnknownFieldMatcher = regexp.MustCompile(`unknown field "([^"]+)"`) ) // LoadDeploymentConfiguration loads the default runtime deployment configuration. func (s *Store) LoadDeploymentConfiguration(ctx context.Context) error { return s.LoadDeploymentConfigurationDir(ctx, DefaultDeploymentConfigurationDir) } // LoadDeploymentConfigurationDir loads and atomically publishes runtime configuration from dir. func (s *Store) LoadDeploymentConfigurationDir(ctx context.Context, dir string) error { entries, err := os.ReadDir(dir) if err != nil { if os.IsNotExist(err) { s.setDeploymentConfiguration(newDeploymentConfiguration()) return nil } return errors.Wrap(err, "failed to read deployment configuration directory") } config := newDeploymentConfiguration() idpFiles := map[string]string{} settingFiles := map[storepb.InstanceSettingKey]string{} for _, entry := range entries { name := entry.Name() path := filepath.Join(dir, name) switch { case isIdentityProviderDeploymentFilename(name): if !idpDeploymentFilenameMatcher.MatchString(name) { slog.Warn("loading identity provider deployment file with a legacy filename; rename it to lowercase kebab case", "filename", name) } provider := &storepb.IdentityProvider{} if err := readDeploymentProtoJSON(path, provider); err != nil { return errors.Wrapf(err, "invalid identity provider deployment file %q", name) } if err := validateDeploymentIdentityProvider(provider); err != nil { return errors.Wrapf(err, "invalid identity provider deployment file %q", name) } if previous, ok := idpFiles[provider.Uid]; ok { return errors.Errorf("identity provider UID %q is declared by both %q and %q", provider.Uid, previous, name) } idpFiles[provider.Uid] = name config.identityProviders[provider.Uid] = cloneIdentityProvider(provider) case instanceSettingDeploymentFilenameMatcher.MatchString(name): setting := &storepb.InstanceSetting{} if err := readDeploymentProtoJSON(path, setting); err != nil { return errors.Wrapf(err, "invalid instance setting deployment file %q", name) } if err := validateAndNormalizeDeploymentInstanceSetting(setting); err != nil { return errors.Wrapf(err, "invalid instance setting deployment file %q", name) } if previous, ok := settingFiles[setting.Key]; ok { return errors.Errorf("instance setting key %q is declared by both %q and %q", setting.Key, previous, name) } settingFiles[setting.Key] = name config.instanceSettings[setting.Key] = cloneInstanceSetting(setting) case strings.HasPrefix(name, "memos-"): slog.Warn("ignoring unrecognized Memos deployment configuration filename", "filename", name) default: // The directory may contain unrelated platform secret files. } } if err := s.validateDeploymentAuthenticationState(ctx, config); err != nil { return err } if err := s.warnShadowedStoredIdentityProviders(ctx, config); err != nil { return err } s.setDeploymentConfiguration(config) slog.Info("loaded deployment configuration", "identityProviders", len(config.identityProviders), "instanceSettings", len(config.instanceSettings)) return nil } func newDeploymentConfiguration() *deploymentConfiguration { return &deploymentConfiguration{ identityProviders: map[string]*storepb.IdentityProvider{}, instanceSettings: map[storepb.InstanceSettingKey]*storepb.InstanceSetting{}, } } func isIdentityProviderDeploymentFilename(name string) bool { // The original database-writing bootstrap accepted every filename with this // prefix and suffix. Continue loading those names so an upgrade cannot // silently fall back to stale credentials stored in the database. return strings.HasPrefix(name, "memos-idp-") && strings.HasSuffix(name, ".json") } func readDeploymentProtoJSON(path string, message proto.Message) error { info, err := os.Stat(path) if err != nil { return errors.Wrap(err, "failed to inspect file") } if !info.Mode().IsRegular() { return errors.New("file must resolve to a regular file") } file, err := os.Open(path) if err != nil { return errors.Wrap(err, "failed to open file") } defer file.Close() info, err = file.Stat() if err != nil { return errors.Wrap(err, "failed to inspect file") } if !info.Mode().IsRegular() { return errors.New("file must resolve to a regular file") } content, err := io.ReadAll(io.LimitReader(file, maxDeploymentConfigurationSize+1)) if err != nil { return errors.Wrap(err, "failed to read file") } if len(content) > maxDeploymentConfigurationSize { return errors.Errorf("file exceeds %d bytes", maxDeploymentConfigurationSize) } if err := (protojson.UnmarshalOptions{DiscardUnknown: false}).Unmarshal(content, message); err != nil { if matches := protoJSONUnknownFieldMatcher.FindStringSubmatch(err.Error()); len(matches) == 2 { return errors.Errorf("failed to decode protobuf JSON: unknown field %q", matches[1]) } return errors.New("failed to decode protobuf JSON; verify field names, value types, and JSON syntax") } return nil } func validateDeploymentIdentityProvider(provider *storepb.IdentityProvider) error { if provider.Id != 0 { return errors.New("id must be omitted") } if !base.UIDMatcher.MatchString(provider.Uid) { return errors.New("uid is invalid") } if strings.TrimSpace(provider.Name) == "" { return errors.New("name is required") } if provider.Type != storepb.IdentityProvider_OAUTH2 { return errors.New("type must be OAUTH2") } if provider.IdentifierFilter != "" { if _, err := regexp.Compile(provider.IdentifierFilter); err != nil { return errors.Wrap(err, "identifierFilter must be a valid regular expression") } } config := provider.Config.GetOauth2Config() if config == nil { return errors.New("config.oauth2Config is required") } required := []struct { name string value string }{ {name: "clientId", value: config.ClientId}, {name: "clientSecret", value: config.ClientSecret}, {name: "authUrl", value: config.AuthUrl}, {name: "tokenUrl", value: config.TokenUrl}, {name: "userInfoUrl", value: config.UserInfoUrl}, } for _, field := range required { if strings.TrimSpace(field.value) == "" { return errors.Errorf("config.oauth2Config.%s is required", field.name) } } for _, field := range []struct { name string value string }{ {name: "authUrl", value: config.AuthUrl}, {name: "tokenUrl", value: config.TokenUrl}, {name: "userInfoUrl", value: config.UserInfoUrl}, } { parsed, err := url.ParseRequestURI(field.value) if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" { return errors.Errorf("config.oauth2Config.%s must be an absolute HTTP(S) URL", field.name) } } if len(config.Scopes) == 0 { return errors.New("config.oauth2Config.scopes is required") } for i, scope := range config.Scopes { if strings.TrimSpace(scope) == "" { return errors.Errorf("config.oauth2Config.scopes[%d] must not be empty", i) } } if config.FieldMapping == nil || strings.TrimSpace(config.FieldMapping.Identifier) == "" { return errors.New("config.oauth2Config.fieldMapping.identifier is required") } return nil } func validateAndNormalizeDeploymentInstanceSetting(setting *storepb.InstanceSetting) error { switch setting.Key { case storepb.InstanceSettingKey_GENERAL: if setting.GetGeneralSetting() == nil { return errors.New("generalSetting must be populated for key GENERAL") } if offset := setting.GetGeneralSetting().WeekStartDayOffset; offset < -1 || offset > 6 { return errors.New("generalSetting.weekStartDayOffset must be between -1 and 6") } case storepb.InstanceSettingKey_STORAGE: storage := setting.GetStorageSetting() if storage == nil { return errors.New("storageSetting must be populated for key STORAGE") } // Normalization would silently self-heal this misconfiguration to LOCAL; // a deployment file declaring S3 without a config should fail loudly. if storage.StorageType == storepb.InstanceStorageSetting_S3 && storage.S3Config == nil && len(storage.Storages) == 0 { return errors.New("storageSetting.s3Config is required for S3") } NormalizeInstanceStorageSetting(storage) if storage.UploadSizeLimitMb < 0 { return errors.New("storageSetting.uploadSizeLimitMb must not be negative") } defaultStorage := GetDefaultStorage(storage) if defaultStorage != nil && defaultStorage.Type == storepb.StorageType_STORAGE_TYPE_S3 { s3Config := defaultStorage.GetS3Config() if s3Config == nil { return errors.New("storageSetting default storage S3 config is required") } for _, field := range []struct { name string value string }{ {name: "accessKeyId", value: s3Config.AccessKeyId}, {name: "accessKeySecret", value: s3Config.AccessKeySecret}, {name: "endpoint", value: s3Config.Endpoint}, {name: "region", value: s3Config.Region}, {name: "bucket", value: s3Config.Bucket}, } { if strings.TrimSpace(field.value) == "" { return errors.Errorf("storageSetting default S3 config.%s is required", field.name) } } } case storepb.InstanceSettingKey_MEMO_RELATED: if setting.GetMemoRelatedSetting() == nil { return errors.New("memoRelatedSetting must be populated for key MEMO_RELATED") } case storepb.InstanceSettingKey_NOTIFICATION: notification := setting.GetNotificationSetting() if notification == nil { return errors.New("notificationSetting must be populated for key NOTIFICATION") } if email := notification.Email; email != nil && email.Enabled { if strings.TrimSpace(email.SmtpHost) == "" || email.SmtpPort <= 0 || strings.TrimSpace(email.FromEmail) == "" { return errors.New("enabled notification email requires smtpHost, a positive smtpPort, and fromEmail") } if email.UseTls && email.UseSsl { return errors.New("notification email cannot enable both useTls and useSsl") } } case storepb.InstanceSettingKey_AI: if setting.GetAiSetting() == nil { return errors.New("aiSetting must be populated for key AI") } if err := normalizeDeploymentAISetting(setting.GetAiSetting()); err != nil { return err } case storepb.InstanceSettingKey_ACCESS: access := setting.GetAccessSetting() if access == nil { return errors.New("accessSetting must be populated for key ACCESS") } switch access.AccessMode { case storepb.InstanceAccessMode_INSTANCE_ACCESS_MODE_PRIVATE, storepb.InstanceAccessMode_INSTANCE_ACCESS_MODE_PUBLIC: default: return errors.New("accessSetting.accessMode must be PRIVATE or PUBLIC") } case storepb.InstanceSettingKey_BASIC, storepb.InstanceSettingKey_TAGS: return errors.Errorf("key %s cannot be deployment configured", setting.Key) default: return errors.Errorf("unsupported instance setting key %s", setting.Key) } return nil } func normalizeDeploymentAISetting(setting *storepb.InstanceAISetting) error { providers := map[string]struct{}{} for i, provider := range setting.Providers { if provider == nil { return errors.Errorf("aiSetting.providers[%d] must not be null", i) } provider.Id = strings.TrimSpace(provider.Id) provider.Title = strings.TrimSpace(provider.Title) provider.Endpoint = strings.TrimSpace(provider.Endpoint) if provider.Id == "" || provider.Title == "" || provider.ApiKey == "" { return errors.Errorf("aiSetting.providers[%d] requires id, title, and apiKey", i) } if _, ok := providers[provider.Id]; ok { return errors.Errorf("aiSetting provider ID %q is duplicated", provider.Id) } providers[provider.Id] = struct{}{} switch provider.Type { case storepb.AIProviderType_OPENAI: if provider.Endpoint == "" { provider.Endpoint = "https://api.openai.com/v1" } case storepb.AIProviderType_GEMINI: if provider.Endpoint == "" { provider.Endpoint = "https://generativelanguage.googleapis.com/v1beta" } default: return errors.Errorf("aiSetting provider %q has unsupported type", provider.Id) } } if transcription := setting.Transcription; transcription != nil { transcription.ProviderId = strings.TrimSpace(transcription.ProviderId) transcription.Model = strings.TrimSpace(transcription.Model) transcription.Language = strings.TrimSpace(transcription.Language) transcription.Prompt = strings.TrimSpace(transcription.Prompt) if transcription.ProviderId != "" { if _, ok := providers[transcription.ProviderId]; !ok { return errors.Errorf("aiSetting transcription providerId %q does not reference a provider", transcription.ProviderId) } } if len(transcription.Model) > maxTranscriptionModelLength || len(transcription.Language) > maxTranscriptionLanguageLength || len(transcription.Prompt) > maxTranscriptionPromptLength { return errors.New("aiSetting transcription configuration exceeds a supported length limit") } } return nil } func (s *Store) validateDeploymentAuthenticationState(ctx context.Context, config *deploymentConfiguration) error { _, generalConfigured := config.instanceSettings[storepb.InstanceSettingKey_GENERAL] if !generalConfigured && len(config.identityProviders) == 0 { general, err := s.getRawInstanceSetting(ctx, storepb.InstanceSettingKey_GENERAL.String()) if err != nil { return errors.Wrap(err, "failed to inspect stored GENERAL setting") } if general == nil || !general.GetGeneralSetting().DisallowPasswordAuth { return nil } providers, err := s.listStoredIdentityProviders(ctx, &FindIdentityProvider{}) if err != nil { return errors.Wrap(err, "failed to inspect stored identity providers") } if len(providers) == 0 { slog.Warn("stored configuration disables password authentication for regular users but has no identity provider; unrelated deployment files remain loadable because administrator password sign-in is available") } return nil } general, err := s.getRawInstanceSetting(ctx, storepb.InstanceSettingKey_GENERAL.String()) if err != nil { return errors.Wrap(err, "failed to read stored GENERAL setting") } if configured := config.instanceSettings[storepb.InstanceSettingKey_GENERAL]; configured != nil { general = cloneInstanceSetting(configured) } if general == nil || !general.GetGeneralSetting().DisallowPasswordAuth { return nil } providers, err := s.listStoredIdentityProviders(ctx, &FindIdentityProvider{}) if err != nil { return errors.Wrap(err, "failed to read stored identity providers") } effectiveUIDs := map[string]struct{}{} for _, provider := range providers { effectiveUIDs[provider.Uid] = struct{}{} } for uid := range config.identityProviders { effectiveUIDs[uid] = struct{}{} } if len(effectiveUIDs) == 0 { return errors.New("deployment configuration disables password authentication for regular users but has no effective identity provider") } return nil } func (s *Store) warnShadowedStoredIdentityProviders(ctx context.Context, config *deploymentConfiguration) error { if len(config.identityProviders) == 0 { return nil } providers, err := s.listStoredIdentityProviders(ctx, &FindIdentityProvider{}) if err != nil { return errors.Wrap(err, "failed to inspect stored identity providers") } for _, provider := range providers { if _, ok := config.identityProviders[provider.Uid]; ok { slog.Warn("deployment identity provider shadows a stored provider; the stored configuration remains in the database", "uid", provider.Uid) } } return nil } func (s *Store) setDeploymentConfiguration(config *deploymentConfiguration) { copy := newDeploymentConfiguration() for uid, provider := range config.identityProviders { copy.identityProviders[uid] = cloneIdentityProvider(provider) } for key, setting := range config.instanceSettings { copy.instanceSettings[key] = cloneInstanceSetting(setting) } s.deploymentConfigMu.Lock() s.deploymentConfig = copy s.deploymentConfigMu.Unlock() // A deployment configuration can supply the STORAGE setting. s.resetStorageDriverCache() } // IsIdentityProviderDeploymentConfigured reports whether uid is file-backed. func (s *Store) IsIdentityProviderDeploymentConfigured(uid string) bool { s.deploymentConfigMu.RLock() defer s.deploymentConfigMu.RUnlock() _, ok := s.deploymentConfig.identityProviders[uid] return ok } // IsInstanceSettingDeploymentConfigured reports whether key is file-backed. func (s *Store) IsInstanceSettingDeploymentConfigured(key storepb.InstanceSettingKey) bool { s.deploymentConfigMu.RLock() defer s.deploymentConfigMu.RUnlock() _, ok := s.deploymentConfig.instanceSettings[key] return ok } func (s *Store) getDeploymentIdentityProvider(uid string) *storepb.IdentityProvider { s.deploymentConfigMu.RLock() defer s.deploymentConfigMu.RUnlock() return cloneIdentityProvider(s.deploymentConfig.identityProviders[uid]) } func (s *Store) listDeploymentIdentityProviders() []*storepb.IdentityProvider { s.deploymentConfigMu.RLock() defer s.deploymentConfigMu.RUnlock() providers := make([]*storepb.IdentityProvider, 0, len(s.deploymentConfig.identityProviders)) for _, provider := range s.deploymentConfig.identityProviders { providers = append(providers, cloneIdentityProvider(provider)) } slices.SortFunc(providers, func(a, b *storepb.IdentityProvider) int { return strings.Compare(a.Uid, b.Uid) }) return providers } func (s *Store) getDeploymentInstanceSetting(key storepb.InstanceSettingKey) *storepb.InstanceSetting { s.deploymentConfigMu.RLock() defer s.deploymentConfigMu.RUnlock() return cloneInstanceSetting(s.deploymentConfig.instanceSettings[key]) } func cloneIdentityProvider(provider *storepb.IdentityProvider) *storepb.IdentityProvider { if provider == nil { return nil } cloned := &storepb.IdentityProvider{} proto.Merge(cloned, provider) return cloned } func cloneInstanceSetting(setting *storepb.InstanceSetting) *storepb.InstanceSetting { if setting == nil { return nil } cloned := &storepb.InstanceSetting{} proto.Merge(cloned, setting) return cloned }