fix: [OCSRE-1629] check the registry NATS credentials only at startup

storeOptions checked the NATS credentials and called os.Exit(1) on an
error. ReconnectedCB calls Init, and Init runs storeOptions again. If
a reconnect happened while a rotated .creds file was only partly
written, the service stopped. In the same path, nats.UserCredentials
reads the file once when the option is applied. When this read
failed, the error was ignored and the connection had no credentials.

Now only the first call checks the credentials, at startup. Later
calls do not check the file. For a file, they set the JWT callbacks
without a read. The callbacks read the file on each connect, so a
partly written file fails only that connect attempt, and NATS
retries it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

ocis-pkg/natsjsregistry/registry.go

@@ -10,10 +10,12 @@ import (
 	"os"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	natsjskv "github.com/go-micro/plugins/v4/store/nats-js-kv"
 	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nkeys"
 	"github.com/owncloud/ocis/v2/ocis-pkg/generators"
 	"github.com/owncloud/reva/v2/pkg/natsauth"
 	revastore "github.com/owncloud/reva/v2/pkg/store"
@@ -75,6 +77,9 @@ type storeregistry struct {
 	typ        string
 	defaultTTL time.Duration
 	lock       sync.RWMutex
+
+	// credsChecked is true after the first check of the NATS credentials.
+	credsChecked atomic.Bool
 }
 
 // Init inits the registry
@@ -221,7 +226,7 @@ func (n *storeregistry) storeOptions(opts registry.Options) []store.Option {
 	natsOptions := nats.GetDefaultOptions()
 	natsOptions.Name = generators.GenerateConnectionName(serviceName, generators.NTypeRegistry)
 	natsOptions.User, natsOptions.Password = getAuth()
-	credsOpt, err := natsauth.CredentialsOption(getCreds())
+	credsOpt, err := n.credentialsOption()
 	if err != nil {
 		fmt.Println("cannot use the nats credentials of the registry:", err)
 		os.Exit(1)
@@ -257,6 +262,56 @@ func getAuth() (string, string) {
 	return os.Getenv(_registryUsernameEnv), os.Getenv(_registryPasswordEnv)
 }
 
+// credentialsOption returns the nats.Option for the JWT credentials of the registry.
+//
+// Only the first call checks the credentials, at startup. An invalid value then stops
+// the service. Later calls happen in Init, for example after a reconnect. A rotated
+// .creds file can then be partly written. So later calls do not check the file and do
+// not read it. The connection reads the file on each connect and retries on an error.
+func (n *storeregistry) credentialsOption() (nats.Option, error) {
+	credsFile, creds := getCreds()
+	if !n.credsChecked.Swap(true) {
+		return natsauth.CredentialsOption(credsFile, creds)
+	}
+	switch {
+	case creds != "":
+		// the content cannot change at runtime, it was checked at startup
+		return nats.UserCredentialBytes([]byte(creds)), nil
+	case credsFile != "":
+		return lazyCredsFileOption(credsFile), nil
+	default:
+		return nil, nil
+	}
+}
+
+// lazyCredsFileOption sets the JWT callbacks for a .creds file. Unlike nats.UserCredentials,
+// it does not read the file when the option is applied. The callbacks read the file on
+// each connect.
+func lazyCredsFileOption(credsFile string) nats.Option {
+	return func(o *nats.Options) error {
+		o.UserJWT = func() (string, error) {
+			content, err := os.ReadFile(credsFile)
+			if err != nil {
+				return "", err
+			}
+			return nkeys.ParseDecoratedJWT(content)
+		}
+		o.SignatureCB = func(nonce []byte) ([]byte, error) {
+			content, err := os.ReadFile(credsFile)
+			if err != nil {
+				return nil, err
+			}
+			kp, err := nkeys.ParseDecoratedUserNKey(content)
+			if err != nil {
+				return nil, err
+			}
+			defer kp.Wipe()
+			return kp.Sign(nonce)
+		}
+		return nil
+	}
+}
+
 // getCreds returns the path and the content of the NATS .creds file for the registry.
 // The registry variables take precedence. If both are empty, the global OCIS_NATS_* variables are used.
 func getCreds() (string, string) {

ocis-pkg/natsjsregistry/registry_creds_test.go

@@ -1,8 +1,13 @@
 package natsjsregistry
 
 import (
+	"os"
+	"path/filepath"
 	"testing"
 
+	"github.com/nats-io/jwt/v2"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nkeys"
 	"github.com/stretchr/testify/require"
 )
 
@@ -38,3 +43,83 @@ func TestGetCreds(t *testing.T) {
 		require.Equal(t, "registry", creds)
 	})
 }
+
+func newTestCreds(t *testing.T) []byte {
+	t.Helper()
+	accountKP, err := nkeys.CreateAccount()
+	require.NoError(t, err)
+	userKP, err := nkeys.CreateUser()
+	require.NoError(t, err)
+	userPub, err := userKP.PublicKey()
+	require.NoError(t, err)
+	userJWT, err := jwt.NewUserClaims(userPub).Encode(accountKP)
+	require.NoError(t, err)
+	seed, err := userKP.Seed()
+	require.NoError(t, err)
+	creds, err := jwt.FormatUserConfig(userJWT, seed)
+	require.NoError(t, err)
+	return creds
+}
+
+func TestCredentialsOptionChecksOnlyOnce(t *testing.T) {
+	creds := newTestCreds(t)
+	credsFile := filepath.Join(t.TempDir(), "user.creds")
+	require.NoError(t, os.WriteFile(credsFile, creds, 0o600))
+	t.Setenv(_natsCredsFileEnv, credsFile)
+
+	n := &storeregistry{}
+
+	// first call: the file is checked
+	opt, err := n.credentialsOption()
+	require.NoError(t, err)
+	require.NotNil(t, opt)
+
+	// a rotation writes the file partly
+	require.NoError(t, os.WriteFile(credsFile, creds[:len(creds)/3], 0o600))
+
+	// later call, for example from Init after a reconnect: no check, no error
+	opt, err = n.credentialsOption()
+	require.NoError(t, err)
+	o := nats.GetDefaultOptions()
+	require.NoError(t, opt(&o), "applying the option must not read the file")
+	require.NotNil(t, o.UserJWT)
+	require.NotNil(t, o.SignatureCB)
+
+	// the connect attempt with the partly written file fails, NATS retries it
+	_, err = o.SignatureCB([]byte("nonce"))
+	require.Error(t, err)
+
+	// a missing file during the rotation also fails only the connect attempt
+	require.NoError(t, os.Remove(credsFile))
+	_, err = o.UserJWT()
+	require.Error(t, err)
+
+	// after the rotation the callbacks use the new file
+	newCreds := newTestCreds(t)
+	require.NoError(t, os.WriteFile(credsFile, newCreds, 0o600))
+	userJWT, err := o.UserJWT()
+	require.NoError(t, err)
+	require.Contains(t, string(newCreds), userJWT)
+	_, err = o.SignatureCB([]byte("nonce"))
+	require.NoError(t, err)
+}
+
+func TestCredentialsOptionFirstCallFails(t *testing.T) {
+	t.Setenv(_natsCredsFileEnv, filepath.Join(t.TempDir(), "missing.creds"))
+	n := &storeregistry{}
+	_, err := n.credentialsOption()
+	require.Error(t, err)
+}
+
+func TestCredentialsOptionContent(t *testing.T) {
+	t.Setenv(_natsCredsEnv, string(newTestCreds(t)))
+	n := &storeregistry{}
+	for i := 0; i < 2; i++ {
+		opt, err := n.credentialsOption()
+		require.NoError(t, err)
+		o := nats.GetDefaultOptions()
+		require.NoError(t, opt(&o))
+		_, err = o.SignatureCB([]byte("nonce"))
+		require.NoError(t, err)
+	}
+}