fix: runtime orchestration fixes

RocketChat/Rocket.Chat#34205

deno-runtime/lib/metricsCollector.ts

@@ -3,6 +3,7 @@ import { Queue } from "./messenger.ts";
 
 export function collectMetrics() {
     return {
+        pid: Deno.pid,
         queueSize: Queue.getCurrentSize(),
     }
 };
@@ -15,6 +16,16 @@ export async function sendMetrics() {
     await writeAll(Deno.stderr, encoder.encode(JSON.stringify(metrics)));
 }
 
-export function startMetricsReport() {
-    setInterval(sendMetrics, 5000);
+let intervalId: number;
+
+export function startMetricsReport(frequencyInMs = 5000) {
+    if (intervalId) {
+        throw new Error('There is already an active metrics report');
+    }
+
+    intervalId = setInterval(sendMetrics, frequencyInMs);
+}
+
+export function abortMetricsReport() {
+    clearInterval(intervalId);
 }

deno-runtime/lib/parseArgs.ts

@@ -0,0 +1,11 @@
+import { parseArgs as $parseArgs } from "https://jsr.io/@std/cli/1.0.9/parse_args.ts";
+
+export type ParsedArgs = {
+    subprocess: string;
+    spawnId: number;
+    metricsReportFrequencyInMs?: number;
+}
+
+export function parseArgs(args: string[]): ParsedArgs {
+    return $parseArgs(args);
+}

deno-runtime/main.ts

@@ -23,6 +23,7 @@ import handleApp from './handlers/app/handler.ts';
 import handleScheduler from './handlers/scheduler-handler.ts';
 import registerErrorListeners from './error-handlers.ts';
 import { startMetricsReport } from './lib/metricsCollector.ts';
+import { parseArgs } from './lib/parseArgs.ts';
 
 type Handlers = {
     app: typeof handleApp;
@@ -128,8 +129,10 @@ async function main() {
     }
 }
 
+const mainArgs = parseArgs(Deno.args);
+
 registerErrorListeners();
 
 main();
 
-startMetricsReport();
+startMetricsReport(mainArgs.metricsReportFrequencyInMs);

src/server/AppManager.ts

@@ -271,7 +271,7 @@ export class AppManager {
                 const prl = new ProxiedApp(this, item, {
                     // Maybe we should have an "EmptyRuntime" class for this?
                     getStatus() {
-                        return AppStatus.COMPILER_ERROR_DISABLED;
+                        return Promise.resolve(AppStatus.COMPILER_ERROR_DISABLED);
                     },
                 } as unknown as DenoRuntimeSubprocessController);
 

src/server/ProxiedApp.ts

@@ -1,4 +1,4 @@
-import type { AppStatus } from '../definition/AppStatus';
+import { AppStatus } from '../definition/AppStatus';
 import { AppsEngineException } from '../definition/exceptions';
 import type { IAppAuthorInfo, IAppInfo } from '../definition/metadata';
 import { AppMethod } from '../definition/metadata';
@@ -80,7 +80,7 @@ export class ProxiedApp {
     }
 
     public async getStatus(): Promise<AppStatus> {
-        return this.appRuntime.getStatus();
+        return this.appRuntime.getStatus().catch(() => AppStatus.UNKNOWN);
     }
 
     public async setStatus(status: AppStatus, silent?: boolean): Promise<void> {

src/server/runtime/deno/AppsEngineDenoRuntime.ts

@@ -97,6 +97,11 @@ export class DenoRuntimeSubprocessController extends EventEmitter {
 
     private state: 'uninitialized' | 'ready' | 'invalid' | 'restarting' | 'unknown' | 'stopped';
 
+    /**
+     * Incremental id that keeps track of how many times we've spawned a process for this app
+     */
+    private spawnId = 0;
+
     private readonly debug: debug.Debugger;
 
     private readonly options = {
@@ -162,6 +167,8 @@ export class DenoRuntimeSubprocessController extends EventEmitter {
                 denoWrapperPath,
                 '--subprocess',
                 this.appPackage.info.id,
+                '--spawnId',
+                String(this.spawnId++),
             ];
 
             this.deno = child_process.spawn(denoExePath, options, { env: { DENO_DIR } });
@@ -303,6 +310,7 @@ export class DenoRuntimeSubprocessController extends EventEmitter {
             logger.info('Successfully restarted app subprocess');
         } catch (e) {
             logger.error("Failed to restart app's subprocess", { error: e.message || e });
+            throw e;
         } finally {
             await this.logStorage.storeEntries(AppConsole.toStorageEntry(this.getAppId(), logger));
         }

src/server/runtime/deno/LivenessManager.ts

@@ -7,10 +7,11 @@ import type { ProcessMessenger } from './ProcessMessenger';
 const COMMAND_PING = '_zPING';
 
 const defaultOptions: LivenessManager['options'] = {
-    pingRequestTimeout: 10000,
+    pingRequestTimeout: 1000,
     pingFrequencyInMS: 10000,
     consecutiveTimeoutLimit: 4,
     maxRestarts: Infinity,
+    restartAttemptDelayInMS: 1000,
 };
 
 /**
@@ -36,6 +37,9 @@ export class LivenessManager {
 
         // Limit of times we can try to restart a process
         maxRestarts: number;
+
+        // Time to delay the next restart attempt after a failed one
+        restartAttemptDelayInMS: number;
     };
 
     private subprocess: ChildProcess;
@@ -80,7 +84,6 @@ export class LivenessManager {
 
         this.pingTimeoutConsecutiveCount = 0;
 
-        this.controller.once('ready', () => this.ping());
         this.subprocess.once('exit', this.handleExit.bind(this));
         this.subprocess.once('error', this.handleError.bind(this));
     }
@@ -198,7 +201,12 @@ export class LivenessManager {
             pid: this.subprocess.pid,
         });
 
-        await this.controller.restartApp();
+        try {
+            await this.controller.restartApp();
+        } catch (e) {
+            this.debug('Restart attempt failed. Retrying in %dms', this.options.restartAttemptDelayInMS);
+            setTimeout(() => this.restartProcess('Failed restart attempt'), this.options.restartAttemptDelayInMS);
+        }
 
         this.pingTimeoutConsecutiveCount = 0;
         this.restartCount++;