diff --git a/cli/command/context/use.go b/cli/command/context/use.go
index 29d157a377..c24a0c6ecb 100644
--- a/cli/command/context/use.go
+++ b/cli/command/context/use.go
@@ -25,20 +25,26 @@ func newUseCommand(dockerCli command.Cli) *cobra.Command {
 
 // RunUse set the current Docker context
 func RunUse(dockerCli command.Cli, name string) error {
-	if err := store.ValidateContextName(name); err != nil && name != "default" {
-		return err
-	}
-	if _, err := dockerCli.ContextStore().GetMetadata(name); err != nil && name != "default" {
-		return err
-	}
-	configValue := name
-	if configValue == "default" {
-		configValue = ""
+	// configValue uses an empty string for "default"
+	var configValue string
+	if name != command.DefaultContextName {
+		if err := store.ValidateContextName(name); err != nil {
+			return err
+		}
+		if _, err := dockerCli.ContextStore().GetMetadata(name); err != nil {
+			return err
+		}
+		configValue = name
 	}
 	dockerConfig := dockerCli.ConfigFile()
-	dockerConfig.CurrentContext = configValue
-	if err := dockerConfig.Save(); err != nil {
-		return err
+	// Avoid updating the config-file if nothing changed. This also prevents
+	// creating the file and config-directory if the default is used and
+	// no config-file existed yet.
+	if dockerConfig.CurrentContext != configValue {
+		dockerConfig.CurrentContext = configValue
+		if err := dockerConfig.Save(); err != nil {
+			return err
+		}
 	}
 	fmt.Fprintln(dockerCli.Out(), name)
 	fmt.Fprintf(dockerCli.Err(), "Current context is now %q\n", name)
diff --git a/cli/command/context/use_test.go b/cli/command/context/use_test.go
index 3053cc03b2..13be26425f 100644
--- a/cli/command/context/use_test.go
+++ b/cli/command/context/use_test.go
@@ -2,6 +2,9 @@ package context
 
 import (
 	"bytes"
+	"errors"
+	"io"
+	"os"
 	"path/filepath"
 	"testing"
 
@@ -10,6 +13,7 @@ import (
 	"github.com/docker/cli/cli/config/configfile"
 	"github.com/docker/cli/cli/context/store"
 	"github.com/docker/cli/cli/flags"
+	"github.com/docker/docker/pkg/homedir"
 	"gotest.tools/v3/assert"
 	is "gotest.tools/v3/assert/cmp"
 )
@@ -46,6 +50,34 @@ func TestUseNoExist(t *testing.T) {
 	assert.Check(t, store.IsErrContextDoesNotExist(err))
 }
 
+// TestUseDefaultWithoutConfigFile verifies that the CLI does not create
+// the default config file and directory when using the default context.
+func TestUseDefaultWithoutConfigFile(t *testing.T) {
+	// We must use a temporary home-directory, because this test covers
+	// the _default_ configuration file. If we specify a custom configuration
+	// file, the CLI produces an error if the file doesn't exist.
+	tmpHomeDir := t.TempDir()
+	t.Setenv(homedir.Key(), tmpHomeDir)
+	configDir := filepath.Join(tmpHomeDir, ".docker")
+	configFilePath := filepath.Join(configDir, "config.json")
+
+	// Verify config-dir and -file don't exist before
+	_, err := os.Stat(configDir)
+	assert.Check(t, errors.Is(err, os.ErrNotExist))
+	_, err = os.Stat(configFilePath)
+	assert.Check(t, errors.Is(err, os.ErrNotExist))
+
+	cli, err := command.NewDockerCli(command.WithCombinedStreams(io.Discard))
+	assert.NilError(t, err)
+	assert.NilError(t, newUseCommand(cli).RunE(nil, []string{"default"}))
+
+	// Verify config-dir and -file don't exist after
+	_, err = os.Stat(configDir)
+	assert.Check(t, errors.Is(err, os.ErrNotExist))
+	_, err = os.Stat(configFilePath)
+	assert.Check(t, errors.Is(err, os.ErrNotExist))
+}
+
 func TestUseHostOverride(t *testing.T) {
 	t.Setenv("DOCKER_HOST", "tcp://ed:2375/")