diff --git a/libxtracfg/java/build.gradle b/libxtracfg/java/build.gradle index f157657..31a52bc 100644 --- a/libxtracfg/java/build.gradle +++ b/libxtracfg/java/build.gradle @@ -27,7 +27,7 @@ repositories { } dependencies { - implementation group: 'de.interactive_instruments', name: 'ldproxy-cfg', version: '4.8.0' + implementation group: 'de.interactive_instruments', name: 'ldproxy-cfg', version: '4.9.0-v5-deprecated-options-migration-SNAPSHOT' implementation group: 'org.slf4j', name: isCi ? 'slf4j-nop' : 'slf4j-simple', version: '2.0.16' // 24.x is for JDK 21 diff --git a/libxtracfg/java/src/main/java/de/ii/xtraplatform/cli/Migration.java b/libxtracfg/java/src/main/java/de/ii/xtraplatform/cli/Migration.java index aa2bcca..ab17a3e 100644 --- a/libxtracfg/java/src/main/java/de/ii/xtraplatform/cli/Migration.java +++ b/libxtracfg/java/src/main/java/de/ii/xtraplatform/cli/Migration.java @@ -40,7 +40,7 @@ private static boolean isMigration(Error vm) { return Objects.equals(vm.getKeyword(), MIGRATION); } - private static final String MIGRATION = "migration"; + static final String MIGRATION = "migration"; static Error migration(String path, String message) { return new Error.Builder() diff --git a/libxtracfg/java/src/main/java/de/ii/xtraplatform/cli/ValueMessages.java b/libxtracfg/java/src/main/java/de/ii/xtraplatform/cli/ValueMessages.java new file mode 100644 index 0000000..c299224 --- /dev/null +++ b/libxtracfg/java/src/main/java/de/ii/xtraplatform/cli/ValueMessages.java @@ -0,0 +1,33 @@ +package de.ii.xtraplatform.cli; + +import java.nio.file.Path; +import java.util.Objects; +import shadow.com.networknt.schema.Error; + +public class ValueMessages extends Messages { + + public ValueMessages(Path path) { + super(null, null, path); + } + + public ValueMessages(Path path, String error) { + super(null, null, path, error); + } + + @Override + protected String getSummary() { + return String.format("Migrations are available for value configuration: %s", getPath()); + } + + @Override + public void log(Result result, boolean verbose) { + if (getError().isPresent() || hasWarnings()) { + super.log(result, verbose); + } + } + + @Override + protected boolean isWarning(Error vm) { + return Objects.equals(vm.getKeyword(), Migration.MIGRATION); + } +} diff --git a/libxtracfg/java/src/main/java/de/ii/xtraplatform/cli/ValuesHandler.java b/libxtracfg/java/src/main/java/de/ii/xtraplatform/cli/ValuesHandler.java new file mode 100644 index 0000000..f274981 --- /dev/null +++ b/libxtracfg/java/src/main/java/de/ii/xtraplatform/cli/ValuesHandler.java @@ -0,0 +1,255 @@ +package de.ii.xtraplatform.cli; + +import de.ii.ldproxy.cfg.LdproxyCfg; +import de.ii.ldproxy.cfg.ValueMigration; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import shadow.com.fasterxml.jackson.databind.JsonNode; +import shadow.com.fasterxml.jackson.databind.ObjectMapper; + +/** Checks and upgrades value files (e.g. stored queries) using the value migrations of ldproxy-cfg. */ +public class ValuesHandler { + + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + + private static class ValueUpgrade { + final Path path; + final Path relativePath; + final boolean isJson; + final JsonNode upgraded; + final ValueMessages messages; + + ValueUpgrade( + Path path, Path relativePath, boolean isJson, JsonNode upgraded, ValueMessages messages) { + this.path = path; + this.relativePath = relativePath; + this.isJson = isJson; + this.upgraded = upgraded; + this.messages = messages; + } + + boolean hasError() { + return messages.getError().isPresent(); + } + + boolean hasUpgrade() { + return Objects.nonNull(upgraded); + } + } + + public static Result check( + LdproxyCfg ldproxyCfg, Optional path, boolean verbose, boolean debug) { + if (Objects.isNull(ldproxyCfg)) { + return Result.failure("Not connected to store"); + } + + Result result = new Result(); + result.details("path", path.orElse("")); + + for (ValueUpgrade upgrade : getUpgrades(ldproxyCfg, path, false, debug)) { + upgrade.messages.log(result, verbose); + } + + return result; + } + + public static Result preUpgrade( + LdproxyCfg ldproxyCfg, Optional path, boolean force, boolean verbose, boolean debug) { + if (Objects.isNull(ldproxyCfg)) { + return Result.failure("Not connected to store"); + } + + Result result = new Result(); + + int i = 0; + for (ValueUpgrade upgrade : getUpgrades(ldproxyCfg, path, force, debug)) { + if (upgrade.hasError()) { + upgrade.messages.logErrors(result, verbose); + } else if (upgrade.hasUpgrade()) { + if (i++ == 0) { + result.info("The following value configurations will be upgraded:"); + } + result.info(" - " + upgrade.relativePath); + } + } + + if (result.has(Result.Status.INFO)) { + result.confirmation("Are you sure?"); + } + + return result; + } + + public static Result upgrade( + LdproxyCfg ldproxyCfg, + Optional path, + boolean doBackup, + boolean force, + boolean verbose, + boolean debug) { + if (Objects.isNull(ldproxyCfg)) { + return Result.failure("Not connected to store"); + } + + Result result = new Result(); + + for (ValueUpgrade upgrade : getUpgrades(ldproxyCfg, path, force, debug)) { + if (upgrade.hasError()) { + result.error( + String.format( + "Could not read %s: %s", upgrade.relativePath, upgrade.messages.getError().get())); + continue; + } + if (!upgrade.hasUpgrade()) { + continue; + } + + if (doBackup) { + Path backup = + upgrade.path.getParent().resolve(upgrade.path.getFileName().toString() + ".backup"); + try { + Files.copy( + upgrade.path, + backup, + StandardCopyOption.REPLACE_EXISTING, + StandardCopyOption.COPY_ATTRIBUTES); + + if (verbose) { + result.success( + String.format( + "Value configuration backup created: %s", + ldproxyCfg.getDataDirectory().relativize(backup))); + } + } catch (IOException e) { + result.error(String.format("Could not create backup %s: %s", backup, e.getMessage())); + continue; + } + } + + try { + if (upgrade.isJson) { + JSON_MAPPER.writerWithDefaultPrettyPrinter().writeValue(upgrade.path.toFile(), upgrade.upgraded); + } else { + ldproxyCfg.getObjectMapper().writeValue(upgrade.path.toFile(), upgrade.upgraded); + } + + result.success(String.format("Value configuration upgraded: %s", upgrade.relativePath)); + } catch (IOException e) { + result.error( + String.format("Could not upgrade %s: %s", upgrade.relativePath, e.getMessage())); + } + } + + return result; + } + + private static List getUpgrades( + LdproxyCfg ldproxyCfg, Optional path, boolean force, boolean debug) { + Map> migrationsByType = + ldproxyCfg.migrations().values().stream() + .collect( + Collectors.groupingBy( + ValueMigration::getValueType, LinkedHashMap::new, Collectors.toList())); + + List upgrades = new ArrayList<>(); + + for (Map.Entry> entry : migrationsByType.entrySet()) { + Path typePath = ldproxyCfg.getValuesPath().resolve(entry.getKey()); + + if (!Files.isDirectory(typePath)) { + continue; + } + + try (Stream files = Files.walk(typePath)) { + files + .filter(Files::isRegularFile) + .filter(ValuesHandler::isValueFile) + .filter( + file -> + path.isEmpty() + || Objects.equals( + Path.of(path.get()).toString(), + ldproxyCfg.getDataDirectory().relativize(file).toString())) + .sorted() + .forEach( + file -> + upgrades.add(getUpgrade(ldproxyCfg, file, entry.getValue(), force, debug))); + } catch (IOException e) { + upgrades.add( + new ValueUpgrade( + typePath, + ldproxyCfg.getDataDirectory().relativize(typePath), + false, + null, + new ValueMessages( + ldproxyCfg.getDataDirectory().relativize(typePath), e.getMessage()))); + } + } + + return upgrades; + } + + private static ValueUpgrade getUpgrade( + LdproxyCfg ldproxyCfg, + Path file, + List migrations, + boolean force, + boolean debug) { + Path relativePath = ldproxyCfg.getDataDirectory().relativize(file); + boolean isJson = isJson(file); + ValueMessages messages = new ValueMessages(relativePath); + + try { + JsonNode original = + isJson + ? JSON_MAPPER.readTree(file.toFile()) + : ldproxyCfg.getObjectMapper().readTree(file.toFile()); + JsonNode upgraded = original; + boolean isUpgraded = false; + + for (ValueMigration migration : migrations) { + if (migration.isApplicable(upgraded)) { + messages.addMessage( + Migration.migration(migration.getSubject(), migration.getDescription())); + upgraded = migration.migrate(upgraded); + isUpgraded = true; + } + } + + if (debug) { + System.out.println("VALUE " + relativePath + " upgraded: " + isUpgraded); + } + + return new ValueUpgrade( + file, relativePath, isJson, isUpgraded || force ? upgraded : null, messages); + } catch (Throwable e) { + if (debug) { + System.err.println("Could not read " + file); + e.printStackTrace(System.err); + } + return new ValueUpgrade( + file, relativePath, isJson, null, new ValueMessages(relativePath, e.getMessage())); + } + } + + private static boolean isValueFile(Path file) { + String name = file.getFileName().toString().toLowerCase(Locale.ROOT); + + return name.endsWith(".json") || name.endsWith(".yml") || name.endsWith(".yaml"); + } + + private static boolean isJson(Path file) { + return file.getFileName().toString().toLowerCase(Locale.ROOT).endsWith(".json"); + } +} diff --git a/libxtracfg/java/src/main/java/de/ii/xtraplatform/cli/cmd/Check.java b/libxtracfg/java/src/main/java/de/ii/xtraplatform/cli/cmd/Check.java index 12e1b1f..938f5fc 100644 --- a/libxtracfg/java/src/main/java/de/ii/xtraplatform/cli/cmd/Check.java +++ b/libxtracfg/java/src/main/java/de/ii/xtraplatform/cli/cmd/Check.java @@ -19,6 +19,8 @@ public Result run(Subcommand cmd, LdproxyCfg ldproxyCfg, Layout layout) { case entities: return EntitiesHandler.check( ldproxyCfg, EntitiesHandler.Type.All, path, ignoreRedundant, verbose, debug); + case values: + return ValuesHandler.check(ldproxyCfg, path, verbose, debug); case layout: return LayoutHandler.check(layout, verbose); default: diff --git a/libxtracfg/java/src/main/java/de/ii/xtraplatform/cli/cmd/Store.java b/libxtracfg/java/src/main/java/de/ii/xtraplatform/cli/cmd/Store.java index 0d7767b..137671a 100644 --- a/libxtracfg/java/src/main/java/de/ii/xtraplatform/cli/cmd/Store.java +++ b/libxtracfg/java/src/main/java/de/ii/xtraplatform/cli/cmd/Store.java @@ -8,6 +8,7 @@ public abstract class Store extends Common { public enum Subcommand { cfg, entities, + values, layout } diff --git a/libxtracfg/java/src/main/java/de/ii/xtraplatform/cli/cmd/Upgrade.java b/libxtracfg/java/src/main/java/de/ii/xtraplatform/cli/cmd/Upgrade.java index 39c97b3..adc32d1 100644 --- a/libxtracfg/java/src/main/java/de/ii/xtraplatform/cli/cmd/Upgrade.java +++ b/libxtracfg/java/src/main/java/de/ii/xtraplatform/cli/cmd/Upgrade.java @@ -28,6 +28,8 @@ public Result run(Subcommand cmd, LdproxyCfg ldproxyCfg, Layout layout) { case entities: return EntitiesHandler.preUpgrade( ldproxyCfg, EntitiesHandler.Type.All, path, ignoreRedundant, force, verbose, debug); + case values: + return ValuesHandler.preUpgrade(ldproxyCfg, path, force, verbose, debug); case layout: return LayoutHandler.preUpgrade(layout, verbose); default: @@ -48,6 +50,8 @@ public Result run(Subcommand cmd, LdproxyCfg ldproxyCfg, Layout layout) { force, verbose, debug); + case values: + return ValuesHandler.upgrade(ldproxyCfg, path, backup, force, verbose, debug); case layout: return LayoutHandler.upgrade(layout, verbose); default: diff --git a/xtracfg/COMMANDS.md b/xtracfg/COMMANDS.md index b75b81f..8b632a1 100644 --- a/xtracfg/COMMANDS.md +++ b/xtracfg/COMMANDS.md @@ -50,6 +50,7 @@ xtracfg check [flags] - [xtracfg](#xtracfg) - [xtracfg check entities](#xtracfg-check-entities) - Check entities in the store source - [xtracfg check layout](#xtracfg-check-layout) - Check layout of the store source +- [xtracfg check values](#xtracfg-check-values) - Check values in the store source ## xtracfg check entities @@ -107,6 +108,38 @@ xtracfg check layout [flags] - [xtracfg check](#xtracfg-check) - Check the store source +## xtracfg check values + +Check values in the store source + +Checks value configurations like stored queries for deprecated settings. +To check only a single value, pass the path to the file relative to the source as argument. + +``` +xtracfg check values [path] [flags] +``` + +#### Examples + +``` +xtracfg check values -v +xtracfg check values -v store/values/queries/api/query.json +``` + +#### Options inherited from parent commands + +``` + -d, --driver string store source driver; currently the only option is FS (default "FS") + --help show help + -r, --ignore-redundant ignore redundant settings + -s, --src string store source (default "./") + -v, --verbose verbose output +``` + +#### See Also + +- [xtracfg check](#xtracfg-check) - Check the store source + ## xtracfg info Print info about the store source @@ -163,6 +196,7 @@ xtracfg upgrade [flags] - [xtracfg](#xtracfg) - [xtracfg upgrade entities](#xtracfg-upgrade-entities) - Upgrade entities in the store source - [xtracfg upgrade layout](#xtracfg-upgrade-layout) - Upgrade layout of the store source +- [xtracfg upgrade values](#xtracfg-upgrade-values) - Upgrade values in the store source ## xtracfg upgrade entities @@ -227,3 +261,39 @@ xtracfg upgrade layout [flags] #### See Also - [xtracfg upgrade](#xtracfg-upgrade) - Upgrade the store source + +## xtracfg upgrade values + +Upgrade values in the store source + +Upgrades value configurations like stored queries with deprecated settings. +To upgrade only a single value, pass the path to the file relative to the source as argument. +No changes are made without confirmation (unless --yes is set). + +``` +xtracfg upgrade values [path] [flags] +``` + +#### Examples + +``` +xtracfg upgrade values -v +xtracfg upgrade values -v store/values/queries/api/query.json +``` + +#### Options inherited from parent commands + +``` + -b, --backup backup files before upgrading + -d, --driver string store source driver; currently the only option is FS (default "FS") + -f, --force upgrade files even if there are no detected issues; useful to harmonize yaml details like quoting and property order + --help show help + -r, --ignore-redundant keep reduntant settings instead of deleting them + -s, --src string store source (default "./") + -v, --verbose verbose output + -y, --yes do not ask for confirmation +``` + +#### See Also + +- [xtracfg upgrade](#xtracfg-upgrade) - Upgrade the store source diff --git a/xtracfg/cmd/store/check.go b/xtracfg/cmd/store/check.go index 4b9b4c1..66eaf0d 100644 --- a/xtracfg/cmd/store/check.go +++ b/xtracfg/cmd/store/check.go @@ -91,6 +91,35 @@ To check only a single entity, pass the path to the file relative to the source }, } + checkValues := &cobra.Command{ + Use: "values [path]", + Short: "Check values in the store source", + Long: `Checks value configurations like stored queries for deprecated settings. +To check only a single value, pass the path to the file relative to the source as argument.`, + Example: name + " check values -v \n" + name + " check values -v store/values/queries/api/query.json", + Args: func(cmd *cobra.Command, args []string) error { + if len(args) > 1 { + return errors.New("only one argument expected") + } + return nil + }, + Run: func(cmd *cobra.Command, args []string) { + if *debug { + fmt.Fprint(os.Stdout, "Checking values in the store source: ", store.Label(), "\n") + } + path := "" + if len(args) > 0 { + path = args[0] + } + + results, err := store.Handle(map[string]interface{}{"ignoreRedundant": strconv.FormatBool(*ignoreRedundant), "path": path}, "check", "values") + + util.PrintResults(results, err) + + printFix(results, err, name) + }, + } + checkLayout := &cobra.Command{ Use: "layout", Short: "Check layout of the store source", @@ -111,6 +140,7 @@ To check only a single entity, pass the path to the file relative to the source check.AddCommand(checkCfg) check.AddCommand(checkEntities) + check.AddCommand(checkValues) check.AddCommand(checkLayout) return check diff --git a/xtracfg/cmd/store/upgrade.go b/xtracfg/cmd/store/upgrade.go index 7f6e08f..9e2aafa 100644 --- a/xtracfg/cmd/store/upgrade.go +++ b/xtracfg/cmd/store/upgrade.go @@ -60,6 +60,20 @@ No changes are made without confirmation (unless --yes is set).`, util.PrintResults(results, err) } + fmt.Fprint(os.Stdout, "\n", "Upgrading values", "\n") + + results, err = store.Handle(map[string]interface{}{"force": strconv.FormatBool(*force)}, "pre_upgrade", "values") + + if !*noConfirm { + util.PrintResults(results, err) + } + + if xtracfg.HasStatus(results, xtracfg.Confirmation) { + results, err = store.Handle(map[string]interface{}{"backup": strconv.FormatBool(*backup), "force": strconv.FormatBool(*force), "noConfirm": strconv.FormatBool(*noConfirm)}, "upgrade", "values") + + util.PrintResults(results, err) + } + fmt.Fprint(os.Stdout, "\n", "Upgrading layout", "\n") results, err = store.Handle(map[string]interface{}{"ignoreRedundant": strconv.FormatBool(*keepRedundant)}, "pre_upgrade", "layout") @@ -156,6 +170,44 @@ No changes are made without confirmation (unless --yes is set).`, }, } + upgradeValues := &cobra.Command{ + Use: "values [path]", + Short: "Upgrade values in the store source", + Long: `Upgrades value configurations like stored queries with deprecated settings. +To upgrade only a single value, pass the path to the file relative to the source as argument. +No changes are made without confirmation (unless --yes is set).`, + Example: name + " upgrade values -v \n" + name + " upgrade values -v store/values/queries/api/query.json", + Args: func(cmd *cobra.Command, args []string) error { + if len(args) > 1 { + return errors.New("only one argument expected") + } + return nil + }, + Run: func(cmd *cobra.Command, args []string) { + if *debug { + fmt.Fprint(os.Stdout, "Upgrading values in the store source: ", store.Label(), "\n") + } + path := "" + if len(args) > 0 { + path = args[0] + } + + results, err := store.Handle(map[string]interface{}{"force": strconv.FormatBool(*force), "path": path}, "pre_upgrade", "values") + + if !*noConfirm { + util.PrintResults(results, err) + } + + if xtracfg.HasStatus(results, xtracfg.Confirmation) { + results, err = store.Handle(map[string]interface{}{"backup": strconv.FormatBool(*backup), "force": strconv.FormatBool(*force), "noConfirm": strconv.FormatBool(*noConfirm), "path": path}, "upgrade", "values") + + util.PrintResults(results, err) + } + + fmt.Fprint(os.Stdout, "\n") + }, + } + upgradeLayout := &cobra.Command{ Use: "layout", Short: "Upgrade layout of the store source", @@ -185,6 +237,7 @@ No changes are made without confirmation (unless --yes is set).`, upgrade.AddCommand(upgradeCfg) upgrade.AddCommand(upgradeEntities) + upgrade.AddCommand(upgradeValues) upgrade.AddCommand(upgradeLayout) return upgrade