keycloak · stianst · Mar 24, 2022 · Mar 15, 2022
@@ -0,0 +1,103 @@
+<#import "/templates/guide.adoc" as tmpl>
+<#import "/templates/kc.adoc" as kc>
+
+<@tmpl.guide
+    title="Importing and Exporting Realms"
+    summary="An overview about how to import and export realms">
+
+In this guide, you are going to understand the different approaches for importing and exporting realms using JSON files.
+
+== Exporting a Realm to a Directory
+
+To export a realm, you can use the `export` command. Your Keycloak server instance must not be started when invoking this command.
+
+<@kc.export parameters="--help"/>
+
+To export a realm to a directory, you can use the `--dir <dir>` option.
+
+<@kc.export parameters="--dir <dir>"/>
+
+When exporting realms to a directory, the server is going to create separate files for each realm being exported.
+
+=== Configuring how users are exported
+
+You are also able to configure how users are going to be exported by setting the `--users <strategy>` option. The values available for this
+option are:
+
+* *different_files*: Users export into different json files, depending on the maximum number of users per file set by `--users-per-file`. This is the default value.
+
+* *skip*: Skips exporting users.
+
+* *realm_file*:  Users will be exported to the same file as the realm settings. For a realm named "foo", this would be "foo-realm.json" with realm data and users.
+
+* *same_file*:  All users are exported to one explicit file. So you will get two json files for a realm, one with realm data and one with users.
+
+If you are exporting users using the `different_files` strategy, you can set how many users per file you want by setting the `--users-per-file` option. The default value is `50`.
+
+<@kc.export parameters="--dir <dir> --users different_files --users-per-file 100"/>
+
+== Exporting a Realm to a File
+
+To export a realm to a file, you can use the `--file <file>` option.
+
+<@kc.export parameters="--file <file>"/>
+
+When exporting realms to a file, the server is going to use the same file to store the configuration for all the realms being exported.
+
+== Exporting a specific realm
+
+If you do not specify a specific realm to export, all realms are exported. To export a single realm, you can use the `--realm` option as follows:
+
+<@kc.export parameters="[--dir|--file] <path> --realm my-realm"/>
+
+== Importing a Realm from a Directory
+
+To import a realm, you can use the `import` command. Your Keycloak server instance must not be started when invoking this command.
+
+<@kc.import parameters="--help"/>
+
+After exporting a realm to a directory, you can use the `--dir <dir>` option to import the realm back to the server as follows:
+
+<@kc.import parameters="--dir <dir>"/>
+
+When importing realms using the `import` command, you are able to set if existing realms should be skipped, or if they should be overridden with the new configuration. For that,
+you can set the `--override` option as follows:
+
+<@kc.import parameters="--dir <dir> --override false"/>
+
+By default, the `--override` option is set to `true` so that realms are always overridden with the new configuration.
+
+== Importing a Realm from a File
+
+To import a realm previously exported in a single file, you can use the `--file <file>` option as follows:
+
+<@kc.import parameters="--file <file>"/>
+
+== Importing a Realm during Startup
+
+You are also able to import realms when the server is starting by using the `--import-realm` option.
+
+<@kc.start parameters="--import-realm"/>
+
+When you set the `--import-realm` option, the server is going to try to import any realm configuration file from the `data/import` directory. Each file in this directory should
+contain a single realm configuration.
+
+If a realm already exists in the server, the import operation is skipped.
+
+== Using Environment Variables within the Realm Configuration Files
+
+When importing a realm, you are able to use placeholders to resolve values from environment variables for any realm configuration.
+
+.Realm configuration using placeholders
+[source, bash]
+----
+{
+    "realm": "${r"${MY_REALM_NAME}"}",
+    "enabled": true,
+    ...
+}
+----
+
+In the example above, the value set to the `MY_REALM_NAME` environment variable is going to be used to set the `realm` property.
+
+</@tmpl.guide>
@@ -17,4 +17,18 @@ bin/kc.[sh|bat]<#if rootParameters?has_content> ${rootParameters}</#if> start<#i
 ----
 bin/kc.[sh|bat] start-dev ${parameters}
 ----
+</#macro>
+
+<#macro export parameters>
+[source,bash]
+----
+bin/kc.[sh|bat] export ${parameters}
+----
+</#macro>
+
+<#macro import parameters>
+[source,bash]
+----
+bin/kc.[sh|bat] import ${parameters}
+----
 </#macro>
@@ -44,8 +44,8 @@
 import java.util.function.UnaryOperator;
 import java.util.stream.Collectors;
 
-import io.quarkus.runtime.Quarkus;
 import org.keycloak.quarkus.runtime.cli.command.Build;
+import org.keycloak.quarkus.runtime.cli.command.ImportRealmMixin;
 import org.keycloak.quarkus.runtime.cli.command.Main;
 import org.keycloak.quarkus.runtime.cli.command.Start;
 import org.keycloak.quarkus.runtime.cli.command.StartDev;
@@ -170,6 +170,7 @@ private static int runReAugmentation(List<String> cliArgs, CommandLine cmd) {
 
         configArgsList.remove(AUTO_BUILD_OPTION_LONG);
         configArgsList.remove(AUTO_BUILD_OPTION_SHORT);
+        configArgsList.remove(ImportRealmMixin.IMPORT_REALM);
 
         configArgsList.replaceAll(new UnaryOperator<String>() {
             @Override

@@ -43,7 +43,7 @@ public final class Export extends AbstractExportImportCommand implements Runnabl
 
     @Option(names = "--realm",
             arity = "1",
-            description = "Set the name of the realm to export",
+            description = "Set the name of the realm to export. If not set, all realms are going to be exported.",
             paramLabel = "<realm>")
     String realm;
 

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2021 Red Hat, Inc. and/or its affiliates
+ * and other contributors as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.keycloak.quarkus.runtime.cli.command;
+
+import static org.keycloak.quarkus.runtime.cli.Picocli.NO_PARAM_LABEL;
+
+import java.io.File;
+import java.util.Optional;
+import org.keycloak.quarkus.runtime.Environment;
+
+import picocli.CommandLine;
+
+public final class ImportRealmMixin {
+
+    public static final String IMPORT_REALM = "--import-realm";
+
+    @CommandLine.Spec
+    private CommandLine.Model.CommandSpec spec;
+
+    @CommandLine.Option(names = IMPORT_REALM,
+            description = "Import realms during startup by reading any realm configuration file from the 'data/import' directory.",
+            paramLabel = NO_PARAM_LABEL,
+            arity = "0")
+    public void setImportRealm(String realmFiles) {
+        StringBuilder filesToImport = new StringBuilder(Optional.ofNullable(realmFiles).orElse(""));
+
+        if (filesToImport.length() > 0) {
+            throw new CommandLine.ParameterException(spec.commandLine(), "Instead of manually specifying the files to import, just copy them to the 'data/import' directory.");
+        }
+
+        File importDir = Environment.getHomePath().resolve("data").resolve("import").toFile();
+
+        if (importDir.exists()) {
+            for (File realmFile : importDir.listFiles()) {
+                filesToImport.append(realmFile.getAbsolutePath()).append(",");
+            }
+        }
+
+        System.setProperty("keycloak.import", filesToImport.toString());
+    }
+}
@@ -50,6 +50,9 @@ public final class Start extends AbstractStartCommand implements Runnable {
             order = 1)
     Boolean autoConfig;
 
+    @CommandLine.Mixin
+    ImportRealmMixin importRealmMixin;
+
     @Override
     protected void doBeforeRun() {
         devProfileNotAllowedError();

@@ -19,6 +19,7 @@
 
 import org.keycloak.quarkus.runtime.Environment;
 
+import picocli.CommandLine;
 import picocli.CommandLine.Command;
 import picocli.CommandLine.Mixin;
 import picocli.CommandLine.Option;
@@ -40,6 +41,9 @@ public final class StartDev extends AbstractStartCommand implements Runnable {
     @Mixin
     HelpAllMixin helpAllMixin;
 
+    @CommandLine.Mixin
+    ImportRealmMixin importRealmMixin;
+
     @Override
     protected void doBeforeRun() {
         Environment.forceDevProfile();

@@ -24,6 +24,8 @@
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
 import java.sql.ResultSet;
@@ -32,6 +34,7 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.StringTokenizer;
 
 import javax.enterprise.inject.Instance;
@@ -49,6 +52,7 @@
 import org.keycloak.Config;
 import org.keycloak.ServerStartupError;
 import org.keycloak.common.Version;
+import org.keycloak.common.util.StringPropertyReplacer;
 import org.keycloak.connections.jpa.DefaultJpaConnectionProvider;
 import org.keycloak.connections.jpa.JpaConnectionProvider;
 import org.keycloak.connections.jpa.updater.JpaUpdaterProvider;
@@ -134,6 +138,8 @@ public void postInit(KeycloakSessionFactory factory) {
 
         if (schemaChanged || Environment.isImportExportMode()) {
             runJobInTransaction(factory, this::initSchema);
+        } else if (System.getProperty("keycloak.import") != null) {
+            importRealms();
         } else {
             //KEYCLOAK-19521 - We should think about a solution which doesn't involve another db lookup in the future.
             MigrationModel model = session.getProvider(DeploymentStateProvider.class).getMigrationModel();
@@ -277,9 +283,15 @@ private void importRealms() {
                 String file = tokenizer.nextToken().trim();
                 RealmRepresentation rep;
                 try {
-                    rep = JsonSerialization.readValue(new FileInputStream(file), RealmRepresentation.class);
-                } catch (Exception e) {
-                    throw new RuntimeException(e);
+                    rep = JsonSerialization.readValue(StringPropertyReplacer.replaceProperties(
+                            Files.readString(Paths.get(file)), new StringPropertyReplacer.PropertyResolver() {
+                                @Override
+                                public String resolve(String property) {
+                                    return Optional.ofNullable(System.getenv(property)).orElse(null);
+                                }
+                            }), RealmRepresentation.class);
+                } catch (Exception cause) {
+                    throw new RuntimeException("Failed to parse realm configuration file: " + file, cause);
                 }
                 importRealm(rep, "file " + file);
             }
@@ -300,7 +312,7 @@ private void importRealm(RealmRepresentation rep, String from) {
                     exists = true;
                 }
 
-                if (manager.getRealmByName(rep.getRealm()) != null) {
+                if (!exists && manager.getRealmByName(rep.getRealm()) != null) {
                     ServicesLogger.LOGGER.realmExists(rep.getRealm(), from);
                     exists = true;
                 }
@@ -309,10 +321,10 @@ private void importRealm(RealmRepresentation rep, String from) {
                     ServicesLogger.LOGGER.importedRealm(realm.getName(), from);
                 }
                 session.getTransactionManager().commit();
-            } catch (Throwable t) {
+            } catch (Throwable cause) {
                 session.getTransactionManager().rollback();
                 if (!exists) {
-                    ServicesLogger.LOGGER.unableToImportRealm(t, rep.getRealm(), from);
+                    throw new RuntimeException("Failed to import realm: " + rep.getRealm(), cause);
                 }
             }
         } finally {

@@ -265,7 +265,7 @@ private Path prepareDistribution() {
             Path distPath = distRootPath.resolve(distDirName.substring(0, distDirName.lastIndexOf('.')));
 
             if (!inited || (reCreate || !distPath.toFile().exists())) {
-                distPath.toFile().delete();
+                FileUtils.deleteDirectory(distPath.toFile());
                 ZipUtils.unzip(distFile.toPath(), distRootPath);
             }
 
@@ -319,8 +319,6 @@ private void startServer(List<String> arguments) throws Exception {
         builder.environment().put("KEYCLOAK_ADMIN", "admin");
         builder.environment().put("KEYCLOAK_ADMIN_PASSWORD", "admin");
 
-        FileUtils.deleteDirectory(distPath.resolve("data").toFile());
-
         keycloak = builder.start();
     }
 
@@ -347,6 +345,8 @@ public void deleteQuarkusProperties() {
     public void copyOrReplaceFileFromClasspath(String file, Path targetFile) {
         File targetDir = distPath.resolve(targetFile).toFile();
 
+        targetDir.mkdirs();
+
         try {
             Files.copy(getClass().getResourceAsStream(file), targetDir.toPath(), StandardCopyOption.REPLACE_EXISTING);
         } catch (IOException cause) {