feat: Allow for custom CLI and variable options

* feat: (AppRunner) Mechanism to configure cli args and derive componentsjs vars from them implemented

* fix: (AppRunner) tidying

* fix: (AppRunner) tidying up

* fix: (AppRunner) runCli method made sync

* fix; (VarResolver) refactored to multiple files, and other stylistic fixes.

* chore: (AppRunner) Uses builder pattern for yargs base arguments setup to enable better typescript inference

* fix(AppRunner): refactoring AppRunner and VarResolver

* fix(AppRunner): refactoring AppRunner promise handling

* fix(AppRunner): verror dependency removal

* fix: Simplify CLI error handling

* feat: Use same config for both CLI and app instantiation

* fix: Update typings and imports

* feat: Split VariableResolver behaviour to 2 classes

* feat: Move default value behaviour from CLI to ValueComputers

* test: Add unit tests for new CLI classes

* feat: Integrate new CLI configuration with all default configurations

* feat: Add createApp function to AppRunner

* docs: Update comments in CLI-related classes

* fix: Various fixes and refactors

Co-authored-by: damooo <damodara@protonmail.com>

src/init/AppRunner.ts

@@ -1,152 +1,199 @@
 /* eslint-disable unicorn/no-process-exit */
-
-import type { ReadStream, WriteStream } from 'tty';
-import type { IComponentsManagerBuilderOptions, LogLevel } from 'componentsjs';
+import type { WriteStream } from 'tty';
+import type { IComponentsManagerBuilderOptions } from 'componentsjs';
 import { ComponentsManager } from 'componentsjs';
 import yargs from 'yargs';
+import { LOG_LEVELS } from '../logging/LogLevel';
 import { getLoggerFor } from '../logging/LogUtil';
-import { ensureTrailingSlash, resolveAssetPath, modulePathPlaceholder } from '../util/PathUtil';
+import { createErrorMessage, isError } from '../util/errors/ErrorUtil';
+import { modulePathPlaceholder, resolveAssetPath } from '../util/PathUtil';
 import type { App } from './App';
+import type { CliResolver } from './CliResolver';
+import type { CliArgv, VariableBindings } from './variables/Types';
 
-const defaultConfig = `${modulePathPlaceholder}config/default.json`;
+const DEFAULT_CONFIG = `${modulePathPlaceholder}config/default.json`;
 
-export interface CliParams {
-  loggingLevel: string;
-  port: number;
-  baseUrl?: string;
-  rootFilePath?: string;
-  sparqlEndpoint?: string;
-  showStackTrace?: boolean;
-  podConfigJson?: string;
-}
+const DEFAULT_CLI_RESOLVER = 'urn:solid-server-app-setup:default:CliResolver';
+const DEFAULT_APP = 'urn:solid-server:default:App';
 
+const CORE_CLI_PARAMETERS = {
+  config: { type: 'string', alias: 'c', default: DEFAULT_CONFIG, requiresArg: true },
+  loggingLevel: { type: 'string', alias: 'l', default: 'info', requiresArg: true, choices: LOG_LEVELS },
+  mainModulePath: { type: 'string', alias: 'm', requiresArg: true },
+} as const;
+
+/**
+ * A class that can be used to instantiate and start a server based on a Component.js configuration.
+ */
 export class AppRunner {
   private readonly logger = getLoggerFor(this);
 
   /**
    * Starts the server with a given config.
    * This method can be used to start the server from within another JavaScript application.
+   * Keys of the `variableBindings` object should be Components.js variables.
+   * E.g.: `{ 'urn:solid-server:default:variable:rootFilePath': '.data' }`.
+   *
    * @param loaderProperties - Components.js loader properties.
    * @param configFile - Path to the server config file.
-   * @param variableParams - Variables to pass into the config file.
+   * @param variableBindings - Parameters to pass into the VariableResolver.
    */
   public async run(
     loaderProperties: IComponentsManagerBuilderOptions<App>,
     configFile: string,
-    variableParams: CliParams,
+    variableBindings: VariableBindings,
   ): Promise<void> {
-    const app = await this.createApp(loaderProperties, configFile, variableParams);
+    const app = await this.create(loaderProperties, configFile, variableBindings);
     await app.start();
   }
 
   /**
-   * Starts the server as a command-line application.
-   * Made non-async to lower the risk of unhandled promise rejections.
-   * @param args - Command line arguments.
-   * @param stderr - Standard error stream.
-   */
-  public runCli({
-    argv = process.argv,
-    stderr = process.stderr,
-  }: {
-    argv?: string[];
-    stdin?: ReadStream;
-    stdout?: WriteStream;
-    stderr?: WriteStream;
-  } = {}): void {
-    // Parse the command-line arguments
-    // eslint-disable-next-line no-sync
-    const params = yargs(argv.slice(2))
-      .strict()
-      .usage('node ./bin/server.js [args]')
-      .check((args): boolean => {
-        if (args._.length > 0) {
-          throw new Error(`Unsupported positional arguments: "${args._.join('", "')}"`);
-        }
-        for (const key of Object.keys(args)) {
-          // We have no options that allow for arrays
-          const val = args[key];
-          if (key !== '_' && Array.isArray(val)) {
-            throw new Error(`Multiple values were provided for: "${key}": "${val.join('", "')}"`);
-          }
-        }
-        return true;
-      })
-      .options({
-        baseUrl: { type: 'string', alias: 'b', requiresArg: true },
-        config: { type: 'string', alias: 'c', default: defaultConfig, requiresArg: true },
-        loggingLevel: { type: 'string', alias: 'l', default: 'info', requiresArg: true },
-        mainModulePath: { type: 'string', alias: 'm', requiresArg: true },
-        port: { type: 'number', alias: 'p', default: 3000, requiresArg: true },
-        rootFilePath: { type: 'string', alias: 'f', default: './', requiresArg: true },
-        showStackTrace: { type: 'boolean', alias: 't', default: false },
-        sparqlEndpoint: { type: 'string', alias: 's', requiresArg: true },
-        podConfigJson: { type: 'string', default: './pod-config.json', requiresArg: true },
-      })
-      .parseSync();
-
-    // Gather settings for instantiating the server
-    const loaderProperties: IComponentsManagerBuilderOptions<App> = {
-      mainModulePath: resolveAssetPath(params.mainModulePath),
-      dumpErrorState: true,
-      logLevel: params.loggingLevel as LogLevel,
-    };
-    const configFile = resolveAssetPath(params.config);
-
-    // Create and execute the app
-    this.createApp(loaderProperties, configFile, params)
-      .then(
-        async(app): Promise<void> => app.start(),
-        (error: Error): void => {
-          // Instantiation of components has failed, so there is no logger to use
-          stderr.write(`Error: could not instantiate server from ${configFile}\n`);
-          stderr.write(`${error.stack}\n`);
-          process.exit(1);
-        },
-      ).catch((error): void => {
-        this.logger.error(`Could not start server: ${error}`, { error });
-        process.exit(1);
-      });
-  }
-
-  /**
-   * Creates the main app object to start the server from a given config.
+   * Returns an App object, created with the given config, that can start and stop the Solid server.
+   * Keys of the `variableBindings` object should be Components.js variables.
+   * E.g.: `{ 'urn:solid-server:default:variable:rootFilePath': '.data' }`.
+   *
    * @param loaderProperties - Components.js loader properties.
-   * @param configFile - Path to a Components.js config file.
-   * @param variables - Variables to pass into the config file.
+   * @param configFile - Path to the server config file.
+   * @param variableBindings - Bindings of Components.js variables.
    */
-  public async createApp(
+  public async create(
     loaderProperties: IComponentsManagerBuilderOptions<App>,
     configFile: string,
-    variables: CliParams | Record<string, any>,
+    variableBindings: VariableBindings,
   ): Promise<App> {
-    // Translate command-line parameters if needed
-    if (typeof variables.loggingLevel === 'string') {
-      variables = this.createVariables(variables as CliParams);
-    }
+    // Create a resolver to translate (non-core) CLI parameters into values for variables
+    const componentsManager = await this.createComponentsManager<App>(loaderProperties, configFile);
 
-    // Set up Components.js
-    const componentsManager = await ComponentsManager.build(loaderProperties);
-    await componentsManager.configRegistry.register(configFile);
-
-    // Create the app
-    const app = 'urn:solid-server:default:App';
-    return await componentsManager.instantiate(app, { variables });
+    // Create the application using the translated variable values
+    return componentsManager.instantiate(DEFAULT_APP, { variables: variableBindings });
   }
 
   /**
-   * Translates command-line parameters into Components.js variables.
+   * Starts the server as a command-line application.
+   * Will exit the process on failure.
+   *
+   * Made non-async to lower the risk of unhandled promise rejections.
+   * This is only relevant when this is used to start as a Node.js application on its own,
+   * if you use this as part of your code you probably want to use the async version.
+   *
+   * @param argv - Command line arguments.
+   * @param stderr - Stream that should be used to output errors before the logger is enabled.
    */
-  protected createVariables(params: CliParams): Record<string, any> {
-    return {
-      'urn:solid-server:default:variable:baseUrl':
-        params.baseUrl ? ensureTrailingSlash(params.baseUrl) : `http://localhost:${params.port}/`,
-      'urn:solid-server:default:variable:loggingLevel': params.loggingLevel,
-      'urn:solid-server:default:variable:port': params.port,
-      'urn:solid-server:default:variable:rootFilePath': resolveAssetPath(params.rootFilePath),
-      'urn:solid-server:default:variable:sparqlEndpoint': params.sparqlEndpoint,
-      'urn:solid-server:default:variable:showStackTrace': params.showStackTrace,
-      'urn:solid-server:default:variable:podConfigJson': resolveAssetPath(params.podConfigJson),
+  public runCliSync({ argv, stderr = process.stderr }: { argv?: CliArgv; stderr?: WriteStream }): void {
+    this.runCli(argv).catch((error): never => {
+      stderr.write(createErrorMessage(error));
+      process.exit(1);
+    });
+  }
+
+  /**
+   * Starts the server as a command-line application.
+   * @param argv - Command line arguments.
+   */
+  public async runCli(argv?: CliArgv): Promise<void> {
+    const app = await this.createCli(argv);
+    try {
+      await app.start();
+    } catch (error: unknown) {
+      this.logger.error(`Could not start the server: ${createErrorMessage(error)}`);
+      this.resolveError('Could not start the server', error);
+    }
+  }
+
+  /**
+   * Returns an App object, created by parsing the Command line arguments, that can start and stop the Solid server.
+   * Will exit the process on failure.
+   *
+   * @param argv - Command line arguments.
+   */
+  public async createCli(argv: CliArgv = process.argv): Promise<App> {
+    // Parse only the core CLI arguments needed to load the configuration
+    const yargv = yargs(argv.slice(2))
+      .usage('node ./bin/server.js [args]')
+      .options(CORE_CLI_PARAMETERS)
+      // We disable help here as it would only show the core parameters
+      .help(false);
+
+    const params = await yargv.parse();
+
+    const loaderProperties = {
+      mainModulePath: resolveAssetPath(params.mainModulePath),
+      dumpErrorState: true,
+      logLevel: params.loggingLevel,
     };
+
+    const config = resolveAssetPath(params.config);
+
+    // Create the Components.js manager used to build components from the provided config
+    let componentsManager: ComponentsManager<any>;
+    try {
+      componentsManager = await this.createComponentsManager(loaderProperties, config);
+    } catch (error: unknown) {
+      // Print help of the expected core CLI parameters
+      const help = await yargv.getHelp();
+      this.resolveError(`${help}\n\nCould not build the config files from ${config}`, error);
+    }
+
+    // Build the CLI components and use them to generate values for the Components.js variables
+    const variables = await this.resolveVariables(componentsManager, argv);
+
+    // Build and start the actual server application using the generated variable values
+    return await this.createApp(componentsManager, variables);
+  }
+
+  /**
+   * Creates the Components Manager that will be used for instantiating.
+   */
+  public async createComponentsManager<T>(
+    loaderProperties: IComponentsManagerBuilderOptions<T>,
+    configFile: string,
+  ): Promise<ComponentsManager<T>> {
+    const componentsManager = await ComponentsManager.build(loaderProperties);
+    await componentsManager.configRegistry.register(configFile);
+    return componentsManager;
+  }
+
+  /**
+   * Handles the first Components.js instantiation,
+   * where CLI settings and variable mappings are created.
+   */
+  private async resolveVariables(componentsManager: ComponentsManager<CliResolver>, argv: string[]):
+  Promise<VariableBindings> {
+    try {
+      // Create a CliResolver, which combines a CliExtractor and a VariableResolver
+      const resolver = await componentsManager.instantiate(DEFAULT_CLI_RESOLVER, {});
+      // Convert CLI args to CLI bindings
+      const cliValues = await resolver.cliExtractor.handleSafe(argv);
+      // Convert CLI bindings into variable bindings
+      return await resolver.settingsResolver.handleSafe(cliValues);
+    } catch (error: unknown) {
+      this.resolveError(`Could not load the config variables`, error);
+    }
+  }
+
+  /**
+   * The second Components.js instantiation,
+   * where the App is created and started using the variable mappings.
+   */
+  private async createApp(componentsManager: ComponentsManager<App>, variables: Record<string, unknown>): Promise<App> {
+    try {
+      // Create the app
+      return await componentsManager.instantiate(DEFAULT_APP, { variables });
+    } catch (error: unknown) {
+      this.resolveError(`Could not create the server`, error);
+    }
+  }
+
+  /**
+   * Throws a new error that provides additional information through the extra message.
+   * Also appends the stack trace to the message.
+   * This is needed for errors that are thrown before the logger is created as we can't log those the standard way.
+   */
+  private resolveError(message: string, error: unknown): never {
+    let errorMessage = `${message}\nCause: ${createErrorMessage(error)}\n`;
+    if (isError(error)) {
+      errorMessage += `${error.stack}\n`;
+    }
+    throw new Error(errorMessage);
   }
 }

src/init/CliResolver.ts

@@ -0,0 +1,16 @@
+import type { CliExtractor } from './cli/CliExtractor';
+import type { SettingsResolver } from './variables/SettingsResolver';
+
+/**
+ * A class that combines a {@link CliExtractor} and a {@link SettingsResolver}.
+ * Mainly exists so both such classes can be generated in a single Components.js instance.
+ */
+export class CliResolver {
+  public readonly cliExtractor: CliExtractor;
+  public readonly settingsResolver: SettingsResolver;
+
+  public constructor(cliExtractor: CliExtractor, settingsResolver: SettingsResolver) {
+    this.cliExtractor = cliExtractor;
+    this.settingsResolver = settingsResolver;
+  }
+}

src/init/cli/CliExtractor.ts

@@ -0,0 +1,18 @@
+import { AsyncHandler } from '../../util/handlers/AsyncHandler';
+import type { CliArgv, Settings } from '../variables/Types';
+
+/**
+ * Converts the input CLI arguments into an easily parseable key/value object.
+ *
+ * Due to how the application is built, there are certain CLI parameters
+ * that need to be parsed before this class can be instantiated.
+ * These can be ignored by this class as they will have been handled before it is called,
+ * but that does mean that this class should not error if they are present,
+ * e.g., by being strict throwing an error on these unexpected parameters.
+ *
+ * The following core CLI parameters are mandatory:
+ *  - -c / \--config
+ *  - -m / \--mainModulePath
+ *  - -l / \--loggingLevel
+ */
+export abstract class CliExtractor extends AsyncHandler<CliArgv, Settings> {}

src/init/cli/YargsCliExtractor.ts

@@ -0,0 +1,61 @@
+/* eslint-disable tsdoc/syntax */
+import type { Arguments, Argv, Options } from 'yargs';
+import yargs from 'yargs';
+import { CliExtractor } from './CliExtractor';
+
+export type YargsArgOptions = Record<string, Options>;
+
+export interface CliOptions {
+  // Usage string printed in case of CLI errors
+  usage?: string;
+  // Errors on unknown CLI parameters when enabled.
+  // @see https://yargs.js.org/docs/#api-reference-strictenabledtrue
+  strictMode?: boolean;
+  // Loads CLI args from environment variables when enabled.
+  // @see http://yargs.js.org/docs/#api-reference-envprefix
+  loadFromEnv?: boolean;
+  // Prefix to be used when `loadFromEnv` is enabled.
+  // @see http://yargs.js.org/docs/#api-reference-envprefix
+  envVarPrefix?: string;
+}
+
+/**
+ * Parses CLI args using the yargs library.
+ * Specific settings can be enabled through the provided options.
+ */
+export class YargsCliExtractor extends CliExtractor {
+  protected readonly yargsArgOptions: YargsArgOptions;
+  protected readonly yargvOptions: CliOptions;
+
+  /**
+   * @param parameters - Parameters that should be parsed from the CLI. @range {json}
+   *                     Format details can be found at https://yargs.js.org/docs/#api-reference-optionskey-opt
+   * @param options - Additional options to configure yargs. @range {json}
+   */
+  public constructor(parameters: YargsArgOptions = {}, options: CliOptions = {}) {
+    super();
+    this.yargsArgOptions = parameters;
+    this.yargvOptions = options;
+  }
+
+  public async handle(argv: readonly string[]): Promise<Arguments> {
+    return this.createYArgv(argv).parse();
+  }
+
+  /**
+   * Creates the yargs Argv object based on the input CLI argv.
+   */
+  private createYArgv(argv: readonly string[]): Argv {
+    let yArgv = yargs(argv.slice(2));
+    if (this.yargvOptions.usage !== undefined) {
+      yArgv = yArgv.usage(this.yargvOptions.usage);
+    }
+    if (this.yargvOptions.strictMode) {
+      yArgv = yArgv.strict();
+    }
+    if (this.yargvOptions.loadFromEnv) {
+      yArgv = yArgv.env(this.yargvOptions.envVarPrefix ?? '');
+    }
+    return yArgv.options(this.yargsArgOptions);
+  }
+}

src/init/variables/CombinedSettingsResolver.ts

@@ -0,0 +1,27 @@
+import { createErrorMessage } from '../../util/errors/ErrorUtil';
+import type { SettingsExtractor } from './extractors/SettingsExtractor';
+import { SettingsResolver } from './SettingsResolver';
+
+/**
+ * Generates variable values by running a set of {@link SettingsExtractor}s on the input.
+ */
+export class CombinedSettingsResolver extends SettingsResolver {
+  public readonly computers: Record<string, SettingsExtractor>;
+
+  public constructor(computers: Record<string, SettingsExtractor>) {
+    super();
+    this.computers = computers;
+  }
+
+  public async handle(input: Record<string, unknown>): Promise<Record<string, unknown>> {
+    const vars: Record<string, any> = {};
+    for (const [ name, computer ] of Object.entries(this.computers)) {
+      try {
+        vars[name] = await computer.handleSafe(input);
+      } catch (err: unknown) {
+        throw new Error(`Error in computing value for variable ${name}: ${createErrorMessage(err)}`);
+      }
+    }
+    return vars;
+  }
+}

src/init/variables/SettingsResolver.ts

@@ -0,0 +1,9 @@
+import { AsyncHandler } from '../../util/handlers/AsyncHandler';
+import type { Settings, VariableBindings } from './Types';
+
+/**
+ * Converts a key/value object, extracted from the CLI or passed as a parameter,
+ * into a new key/value object where the keys are variables defined in the Components.js configuration.
+ * The resulting values are the values that should be assigned to those variables.
+ */
+export abstract class SettingsResolver extends AsyncHandler<Settings, VariableBindings> {}

src/init/variables/Types.ts

@@ -0,0 +1,16 @@
+// These types are used to clarify what is expected for the CLI-related handlers
+
+/**
+ * A list of command line arguments provided to the process.
+ */
+export type CliArgv = string[];
+
+/**
+ * A key/value mapping of parsed command line arguments.
+ */
+export type Settings = Record<string, unknown>;
+
+/**
+ * A key/value mapping of Components.js variables.
+ */
+export type VariableBindings = Record<string, unknown>;

src/init/variables/extractors/AssetPathExtractor.ts

@@ -0,0 +1,26 @@
+import { resolveAssetPath } from '../../../util/PathUtil';
+import type { Settings } from '../Types';
+import { SettingsExtractor } from './SettingsExtractor';
+
+/**
+ * A {@link SettingsExtractor} that converts a path value to an absolute asset path by making use of `resolveAssetPath`.
+ * Returns the default path in case it is defined and no path was found in the map.
+ */
+export class AssetPathExtractor extends SettingsExtractor {
+  private readonly key: string;
+  private readonly defaultPath?: string;
+
+  public constructor(key: string, defaultPath?: string) {
+    super();
+    this.key = key;
+    this.defaultPath = defaultPath;
+  }
+
+  public async handle(args: Settings): Promise<unknown> {
+    const path = args[this.key] ?? this.defaultPath;
+    if (typeof path !== 'string') {
+      throw new Error(`Invalid ${this.key} argument`);
+    }
+    return resolveAssetPath(path);
+  }
+}

src/init/variables/extractors/BaseUrlExtractor.ts

@@ -0,0 +1,24 @@
+import { ensureTrailingSlash } from '../../../util/PathUtil';
+import type { Settings } from '../Types';
+import { SettingsExtractor } from './SettingsExtractor';
+
+/**
+ * A {@link SettingsExtractor} that that generates the base URL based on the input `baseUrl` value,
+ * or by using the port if the first isn't provided.
+ */
+export class BaseUrlExtractor extends SettingsExtractor {
+  private readonly defaultPort: number;
+
+  public constructor(defaultPort = 3000) {
+    super();
+    this.defaultPort = defaultPort;
+  }
+
+  public async handle(args: Settings): Promise<unknown> {
+    if (typeof args.baseUrl === 'string') {
+      return ensureTrailingSlash(args.baseUrl);
+    }
+    const port = args.port ?? this.defaultPort;
+    return `http://localhost:${port}/`;
+  }
+}

src/init/variables/extractors/KeyExtractor.ts

@@ -0,0 +1,21 @@
+import type { Settings } from '../Types';
+import { SettingsExtractor } from './SettingsExtractor';
+
+/**
+ * A simple {@link SettingsExtractor} that extracts a single value from the input map.
+ * Returns the default value if it was defined in case no value was found in the map.
+ */
+export class KeyExtractor extends SettingsExtractor {
+  private readonly key: string;
+  private readonly defaultValue: unknown;
+
+  public constructor(key: string, defaultValue?: unknown) {
+    super();
+    this.key = key;
+    this.defaultValue = defaultValue;
+  }
+
+  public async handle(args: Settings): Promise<unknown> {
+    return typeof args[this.key] === 'undefined' ? this.defaultValue : args[this.key];
+  }
+}

src/init/variables/extractors/SettingsExtractor.ts

@@ -0,0 +1,7 @@
+import { AsyncHandler } from '../../../util/handlers/AsyncHandler';
+import type { Settings } from '../Types';
+
+/**
+ * A handler that computes a specific value from a given map of values.
+ */
+export abstract class SettingsExtractor extends AsyncHandler<Settings, unknown> {}