From 6b432062813a90c21db1a0a3587498e9e479985e Mon Sep 17 00:00:00 2001
From: Robert Kaussow <mail@thegeeklab.de>
Date: Mon, 17 Mar 2025 07:16:16 +0100
Subject: [PATCH] Add backoff retry for store setup (#4964)

---
 cmd/server/flags.go  |  6 ++++++
 cmd/server/server.go | 15 +++++++++++++--
 cmd/server/setup.go  |  4 ++++
 3 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/cmd/server/flags.go b/cmd/server/flags.go
index 97fd8b389..fb01d2918 100644
--- a/cmd/server/flags.go
+++ b/cmd/server/flags.go
@@ -57,6 +57,12 @@ var flags = append([]cli.Flag{
 		Usage:   "time an active connection is allowed to stay open",
 		Value:   3 * time.Second,
 	},
+	&cli.UintFlag{
+		Sources: cli.EnvVars("WOODPECKER_DATABASE_MAX_RETRIES"),
+		Name:    "db-max-retries",
+		Usage:   "max number of retries for the initial connection to the database",
+		Value:   10,
+	},
 	&cli.StringFlag{
 		Sources: cli.EnvVars("WOODPECKER_HOST", "CROW_HOST"),
 		Name:    "server-host",
diff --git a/cmd/server/server.go b/cmd/server/server.go
index 0b3cf5047..775d0e948 100644
--- a/cmd/server/server.go
+++ b/cmd/server/server.go
@@ -25,6 +25,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/cenkalti/backoff/v5"
 	"github.com/gin-gonic/gin"
 	prometheus_http "github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/rs/zerolog"
@@ -36,6 +37,7 @@ import (
 	"codeberg.org/crowci/crow/v3/server/cron"
 	"codeberg.org/crowci/crow/v3/server/router"
 	"codeberg.org/crowci/crow/v3/server/router/middleware"
+	"codeberg.org/crowci/crow/v3/server/store"
 	"codeberg.org/crowci/crow/v3/server/web"
 	"codeberg.org/crowci/crow/v3/shared/logger"
 	"codeberg.org/crowci/crow/v3/version"
@@ -91,10 +93,19 @@ func run(ctx context.Context, c *cli.Command) error {
 		)
 	}
 
-	_store, err := setupStore(ctx, c)
+	_store, err := backoff.Retry(ctx,
+		func() (store.Store, error) {
+			return setupStore(ctx, c)
+		},
+		backoff.WithBackOff(backoff.NewExponentialBackOff()),
+		backoff.WithMaxTries(uint(c.Uint("db-max-retries"))),
+		backoff.WithNotify(func(err error, delay time.Duration) {
+			log.Error().Msgf("failed to setup store: %v: retry in %v", err, delay)
+		}))
 	if err != nil {
-		return fmt.Errorf("can't setup store: %w", err)
+		return err
 	}
+
 	defer func() {
 		if err := _store.Close(); err != nil {
 			log.Error().Err(err).Msg("could not close store")
diff --git a/cmd/server/setup.go b/cmd/server/setup.go
index 3802a2b6e..e9a476448 100644
--- a/cmd/server/setup.go
+++ b/cmd/server/setup.go
@@ -90,6 +90,10 @@ func setupStore(ctx context.Context, c *cli.Command) (store.Store, error) {
 		return nil, fmt.Errorf("could not open datastore: %w", err)
 	}
 
+	if err = store.Ping(); err != nil {
+		return nil, err
+	}
+
 	if err := store.Migrate(ctx, c.Bool("migrations-allow-long")); err != nil {
 		return nil, fmt.Errorf("could not migrate datastore: %w", err)
 	}