Lesson 9: Testing

Tests at two levels: the definition alone, and the workflow running in the engine against PostgreSQL in a container, with a recording transport in place of the HTTP transport. Everything here is your own test code against the public packages; it needs no source of the engine.

A definition test

[Test]
public void TheDefinitionCompilesToAValidCanonicalDefinition()
{
    var compilation = WorkflowCanonicalDefinitionCompiler.Compile(new DocumentReviewWorkflow());

    Assert.That(compilation.Succeeded, Is.True, string.Join("; ", compilation.Diagnostics.Select(d => d.Message)));

    // The engine's validator: the schema, the expressions, and the modules the definition needs.
    using var services = new ServiceCollection()
        .AddLogging()
        .AddWorkflowEngineCoreServices(new ConfigurationBuilder().Build())
        .BuildServiceProvider();
    using var scope = services.CreateScope();
    var validation = scope.ServiceProvider.GetRequiredService<WorkflowCanonicalDefinitionService>()
        .Validate(new WorkflowCanonicalValidateRequest
        {
            CanonicalDefinitionJson = WorkflowCanonicalJsonSerializer.Serialize(compilation.Definition!),
        });

    var issues = validation.SchemaIssues.Concat(validation.SemanticIssues).Concat(validation.ModuleIssues);
    Assert.That(validation.IsValid, Is.True, string.Join("; ", issues.Select(issue => $"{issue.Path}: {issue.Message}")));
}

A recording transport

/// <summary>
/// A test stand-in for the <c>endpoint</c> address kind: it records every call and answers from
/// scripted responses, keyed by service name and path. Register it in place of the HTTP transport.
/// </summary>
public sealed class RecordingEndpointTransport : IWorkflowEndpointTransport
{
    private readonly ConcurrentDictionary<string, WorkflowEndpointResponse> responses = new(StringComparer.OrdinalIgnoreCase);
    private readonly ConcurrentQueue<WorkflowEndpointRequest> calls = new();

    public IReadOnlyCollection<WorkflowEndpointRequest> Calls => calls.ToArray();

    public RecordingEndpointTransport Respond(string serviceName, string path, object? payload)
    {
        responses[Key(serviceName, path)] = new WorkflowEndpointResponse { Succeeded = true, Payload = payload };
        return this;
    }

    public RecordingEndpointTransport Fail(string serviceName, string path, long statusCode, string error)
    {
        responses[Key(serviceName, path)] = new WorkflowEndpointResponse
        {
            Succeeded = false,
            StatusCode = statusCode,
            Error = error,
        };
        return this;
    }

    public Task<WorkflowEndpointResponse> ExecuteAsync(
        WorkflowEndpointRequest request,
        CancellationToken cancellationToken = default)
    {
        calls.Enqueue(request);
        return Task.FromResult(responses.TryGetValue(Key(request.ServiceName, request.Path), out var response)
            ? response
            : new WorkflowEndpointResponse { Succeeded = true });
    }

    private static string Key(string serviceName, string path) => $"{serviceName} {path.TrimStart('/')}";
}

The test host

[OneTimeSetUp]
public async Task StartEngine()
{
    database = new PostgreSqlBuilder().WithImage("postgres:16-alpine").Build();
    try
    {
        await database.StartAsync();
    }
    catch (Exception exception) when (exception is not OperationCanceledException)
    {
        Assert.Ignore($"Docker is not available: {exception.Message}");
    }

    var builder = Host.CreateApplicationBuilder();
    builder.Configuration.AddInMemoryCollection(new Dictionary<string, string?>
    {
        ["STELLAOPS_POSTGRES_WORKFLOW_CONNECTION"] = database.GetConnectionString(),
        ["WorkflowBackend:Provider"] = "Postgres",
        ["WorkflowRuntime:DefaultProvider"] = "Stella.Engine",
    });
    builder.Services.AddWorkflowEngineCoreServices(builder.Configuration);
    builder.Services.AddWorkflowPostgresDataStore(builder.Configuration);
    // The signal pump resumes instances that wait for a child workflow, a timer or a signal.
    builder.Services.AddWorkflowEngineHostedServices();
    // The recording transport answers `endpoint` calls. A definition requires the module of each address
    // kind it uses; a transport package installs it, a test registers it with the stand-in.
    builder.Services.AddWorkflowModule("transport.endpoint", "1.0.0");
    builder.Services.AddScoped<IWorkflowEndpointTransport>(_ => transport);
    builder.Services.AddWorkflowRegistration<DocumentReviewWorkflow, DocumentReviewRequest>();
    builder.Services.AddWorkflowRegistration<SendInvoiceWorkflow, SendInvoiceRequest>();

    host = builder.Build();
    await host.StartAsync(); // applies the schema script, then starts the hosted services
}

A human task, end to end

[Test]
public async Task AChangeRequestOpensTheRevisionTaskAndTheRevisionReturnsToTheReviewer()
{
    using var scope = host!.Services.CreateScope();
    var runtime = scope.ServiceProvider.GetRequiredService<WorkflowRuntimeService>();

    var started = await runtime.StartWorkflowAsync(new StartWorkflowRequest
    {
        WorkflowName = "DocumentReview",
        Payload = new Dictionary<string, object?> { ["documentId"] = "DOC-7", ["title"] = "Travel policy", ["author"] = "author-1" },
    });
    Assert.That(started.NextTasks.Select(task => task.TaskName), Is.EqualTo(new[] { "Review document" }));

    var changes = await runtime.CompleteTaskAsync(new WorkflowTaskCompleteRequest
    {
        WorkflowTaskId = started.NextTasks.Single().WorkflowTaskId,
        ActorId = "reviewer-1",
        ActorRoles = ["reviewer"],
        Payload = new Dictionary<string, object?> { ["decision"] = "changes", ["comments"] = "Add the approval limits." },
    });
    Assert.That(changes.NextTasks.Select(task => task.TaskName), Is.EqualTo(new[] { "Revise document" }));

    var revised = await runtime.CompleteTaskAsync(new WorkflowTaskCompleteRequest
    {
        WorkflowTaskId = changes.NextTasks.Single().WorkflowTaskId,
        ActorId = "author-1",
        ActorRoles = ["author"],
    });
    Assert.That(revised.NextTasks.Select(task => task.TaskName), Is.EqualTo(new[] { "Review document" }));
    Assert.That(revised.WorkflowState["revision"]?.ToString(), Is.EqualTo("2"));
}

Writing the canonical JSON

A small helper writes the canonical JSON of a workflow to a file, to review it, to validate it or to import it (Governed activation and publishing):

/// <summary>
/// Writes the canonical JSON of a workflow to a file: the document to review, to validate, or to import
/// into a workflow host.
/// </summary>
public static void WriteCanonicalJson<TStartRequest>(IDeclarativeWorkflow<TStartRequest> workflow, string path)
    where TStartRequest : class
{
    var compilation = WorkflowCanonicalDefinitionCompiler.Compile(workflow);
    if (!compilation.Succeeded)
    {
        throw new InvalidOperationException(
            string.Join("; ", compilation.Diagnostics.Select(diagnostic => $"{diagnostic.Code}: {diagnostic.Message}")));
    }

    File.WriteAllText(path, WorkflowCanonicalJsonSerializer.Serialize(compilation.Definition!));
}

Counting finished instances

The engine counts every finished instance on the counter workflow.execution.terminal of the meter StellaOps.Workflow, labelled definition and outcome. A MeterListener reads it in a test without an OpenTelemetry pipeline:

/// <summary>
/// Counts the finished workflow instances of this process from the engine's metrics: the counter
/// <c>workflow.execution.terminal</c> of the meter <c>StellaOps.Workflow</c>, labelled
/// <c>definition</c> and <c>outcome</c>. A test, a health probe or a dashboard of your own can read it
/// without an OpenTelemetry pipeline.
/// </summary>
public sealed class FinishedInstanceCounter : IDisposable
{
    private readonly MeterListener listener = new();
    private readonly ConcurrentDictionary<(string Definition, string Outcome), long> counts = new();

    public FinishedInstanceCounter()
    {
        listener.InstrumentPublished = (instrument, meterListener) =>
        {
            if (instrument.Meter.Name == "StellaOps.Workflow" && instrument.Name == "workflow.execution.terminal")
            {
                meterListener.EnableMeasurementEvents(instrument);
            }
        };
        listener.SetMeasurementEventCallback<long>((_, value, tags, _) =>
        {
            string definition = string.Empty, outcome = string.Empty;
            foreach (var tag in tags)
            {
                if (tag.Key == "definition") definition = tag.Value?.ToString() ?? string.Empty;
                if (tag.Key == "outcome") outcome = tag.Value?.ToString() ?? string.Empty;
            }

            counts.AddOrUpdate((definition, outcome), value, (_, total) => total + value);
        });
        listener.Start();
    }

    /// <summary>The instances of a workflow that finished with an outcome, for example <c>Completed</c>.</summary>
    public long Finished(string definition, string outcome = "Completed")
        => counts.TryGetValue((definition, outcome), out var count) ? count : 0;

    public void Dispose() => listener.Dispose();
}

Create the counter before the action and read it after: an AcknowledgeDocument instance counts once, with the outcome Completed, when its task is completed, and not when it starts. The other metrics are listed in Operations.

The complete code

The lesson’s complete code. It uses the namespace of the tutorials; put the classes in a namespace of yours.

DefinitionTests.cs:

using Examples.Workflow.Tutorials.HumanTasks;

using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;

using NUnit.Framework;

using StellaOps.Workflow.Abstractions;
using StellaOps.Workflow.Contracts;
using StellaOps.Workflow.Engine.Services;

namespace Examples.Workflow.Tutorials.Testing;

/// <summary>Tests of a definition: no database, no host.</summary>
[TestFixture]
public sealed class DefinitionTests
{
    [Test]
    public void TheDefinitionCompilesToAValidCanonicalDefinition()
    {
        var compilation = WorkflowCanonicalDefinitionCompiler.Compile(new DocumentReviewWorkflow());

        Assert.That(compilation.Succeeded, Is.True, string.Join("; ", compilation.Diagnostics.Select(d => d.Message)));

        // The engine's validator: the schema, the expressions, and the modules the definition needs.
        using var services = new ServiceCollection()
            .AddLogging()
            .AddWorkflowEngineCoreServices(new ConfigurationBuilder().Build())
            .BuildServiceProvider();
        using var scope = services.CreateScope();
        var validation = scope.ServiceProvider.GetRequiredService<WorkflowCanonicalDefinitionService>()
            .Validate(new WorkflowCanonicalValidateRequest
            {
                CanonicalDefinitionJson = WorkflowCanonicalJsonSerializer.Serialize(compilation.Definition!),
            });

        var issues = validation.SchemaIssues.Concat(validation.SemanticIssues).Concat(validation.ModuleIssues);
        Assert.That(validation.IsValid, Is.True, string.Join("; ", issues.Select(issue => $"{issue.Path}: {issue.Message}")));
    }

    [Test]
    public void ChangesRequestedSendsTheDocumentBackToTheAuthor()
    {
        var definition = WorkflowCanonicalDefinitionCompiler.Compile(new DocumentReviewWorkflow()).Definition!;

        var review = definition.Tasks.Single(task => task.TaskName == "Review document");
        var json = WorkflowCanonicalJsonSerializer.SerializeFragment(review.OnComplete);

        Assert.That(review.TaskRoles, Is.EqualTo(new[] { "reviewer" }));
        Assert.That(review.TimeoutSeconds, Is.EqualTo(259200));
        Assert.That(json, Does.Contain("\"Revise document\""));
    }
}

RecordingEndpointTransport.cs:

using System.Collections.Concurrent;

using StellaOps.Workflow.Abstractions;
using StellaOps.Workflow.Contracts;

namespace Examples.Workflow.Tutorials.Testing;

/// <summary>
/// A test stand-in for the <c>endpoint</c> address kind: it records every call and answers from
/// scripted responses, keyed by service name and path. Register it in place of the HTTP transport.
/// </summary>
public sealed class RecordingEndpointTransport : IWorkflowEndpointTransport
{
    private readonly ConcurrentDictionary<string, WorkflowEndpointResponse> responses = new(StringComparer.OrdinalIgnoreCase);
    private readonly ConcurrentQueue<WorkflowEndpointRequest> calls = new();

    public IReadOnlyCollection<WorkflowEndpointRequest> Calls => calls.ToArray();

    public RecordingEndpointTransport Respond(string serviceName, string path, object? payload)
    {
        responses[Key(serviceName, path)] = new WorkflowEndpointResponse { Succeeded = true, Payload = payload };
        return this;
    }

    public RecordingEndpointTransport Fail(string serviceName, string path, long statusCode, string error)
    {
        responses[Key(serviceName, path)] = new WorkflowEndpointResponse
        {
            Succeeded = false,
            StatusCode = statusCode,
            Error = error,
        };
        return this;
    }

    public Task<WorkflowEndpointResponse> ExecuteAsync(
        WorkflowEndpointRequest request,
        CancellationToken cancellationToken = default)
    {
        calls.Enqueue(request);
        return Task.FromResult(responses.TryGetValue(Key(request.ServiceName, request.Path), out var response)
            ? response
            : new WorkflowEndpointResponse { Succeeded = true });
    }

    private static string Key(string serviceName, string path) => $"{serviceName} {path.TrimStart('/')}";
}

RuntimeTests.cs:

using Examples.Workflow.Tutorials.HumanTasks;
using Examples.Workflow.Tutorials.SubWorkflows;

using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;

using NUnit.Framework;

using StellaOps.Workflow.Abstractions;
using StellaOps.Workflow.Contracts;
using StellaOps.Workflow.DataStore.PostgreSQL;
using StellaOps.Workflow.Engine.Services;

using Testcontainers.PostgreSql;

namespace Examples.Workflow.Tutorials.Testing;

/// <summary>
/// Tests that run workflows in the engine against a PostgreSQL database in a container
/// (Testcontainers), with the recording transport in place of the HTTP transport.
/// </summary>
[TestFixture]
[NonParallelizable]
public sealed class RuntimeTests
{
    private PostgreSqlContainer? database;
    private IHost? host;
    private readonly RecordingEndpointTransport transport = new RecordingEndpointTransport()
        .Respond("billing", "/invoices", new { invoiceId = "INV-1" });

    [OneTimeSetUp]
    public async Task StartEngine()
    {
        database = new PostgreSqlBuilder().WithImage("postgres:16-alpine").Build();
        try
        {
            await database.StartAsync();
        }
        catch (Exception exception) when (exception is not OperationCanceledException)
        {
            Assert.Ignore($"Docker is not available: {exception.Message}");
        }

        var builder = Host.CreateApplicationBuilder();
        builder.Configuration.AddInMemoryCollection(new Dictionary<string, string?>
        {
            ["STELLAOPS_POSTGRES_WORKFLOW_CONNECTION"] = database.GetConnectionString(),
            ["WorkflowBackend:Provider"] = "Postgres",
            ["WorkflowRuntime:DefaultProvider"] = "Stella.Engine",
        });
        builder.Services.AddWorkflowEngineCoreServices(builder.Configuration);
        builder.Services.AddWorkflowPostgresDataStore(builder.Configuration);
        // The signal pump resumes instances that wait for a child workflow, a timer or a signal.
        builder.Services.AddWorkflowEngineHostedServices();
        // The recording transport answers `endpoint` calls. A definition requires the module of each address
        // kind it uses; a transport package installs it, a test registers it with the stand-in.
        builder.Services.AddWorkflowModule("transport.endpoint", "1.0.0");
        builder.Services.AddScoped<IWorkflowEndpointTransport>(_ => transport);
        builder.Services.AddWorkflowRegistration<DocumentReviewWorkflow, DocumentReviewRequest>();
        builder.Services.AddWorkflowRegistration<SendInvoiceWorkflow, SendInvoiceRequest>();

        host = builder.Build();
        await host.StartAsync(); // applies the schema script, then starts the hosted services
    }

    [OneTimeTearDown]
    public async Task StopEngine()
    {
        if (host is not null)
        {
            await host.StopAsync();
            host.Dispose();
        }

        if (database is not null)
        {
            await database.DisposeAsync();
        }
    }

    [Test]
    public async Task AChangeRequestOpensTheRevisionTaskAndTheRevisionReturnsToTheReviewer()
    {
        using var scope = host!.Services.CreateScope();
        var runtime = scope.ServiceProvider.GetRequiredService<WorkflowRuntimeService>();

        var started = await runtime.StartWorkflowAsync(new StartWorkflowRequest
        {
            WorkflowName = "DocumentReview",
            Payload = new Dictionary<string, object?> { ["documentId"] = "DOC-7", ["title"] = "Travel policy", ["author"] = "author-1" },
        });
        Assert.That(started.NextTasks.Select(task => task.TaskName), Is.EqualTo(new[] { "Review document" }));

        var changes = await runtime.CompleteTaskAsync(new WorkflowTaskCompleteRequest
        {
            WorkflowTaskId = started.NextTasks.Single().WorkflowTaskId,
            ActorId = "reviewer-1",
            ActorRoles = ["reviewer"],
            Payload = new Dictionary<string, object?> { ["decision"] = "changes", ["comments"] = "Add the approval limits." },
        });
        Assert.That(changes.NextTasks.Select(task => task.TaskName), Is.EqualTo(new[] { "Revise document" }));

        var revised = await runtime.CompleteTaskAsync(new WorkflowTaskCompleteRequest
        {
            WorkflowTaskId = changes.NextTasks.Single().WorkflowTaskId,
            ActorId = "author-1",
            ActorRoles = ["author"],
        });
        Assert.That(revised.NextTasks.Select(task => task.TaskName), Is.EqualTo(new[] { "Review document" }));
        Assert.That(revised.WorkflowState["revision"]?.ToString(), Is.EqualTo("2"));
    }

    [Test]
    public async Task TheInvoiceWorkflowCallsTheBillingService()
    {
        using var scope = host!.Services.CreateScope();
        var runtime = scope.ServiceProvider.GetRequiredService<WorkflowRuntimeService>();

        var started = await runtime.StartWorkflowAsync(new StartWorkflowRequest
        {
            WorkflowName = "SendInvoice",
            Payload = new Dictionary<string, object?> { ["orderId"] = "ORD-42" },
        });

        var instance = await runtime.GetInstanceAsync(new WorkflowInstanceGetRequest
        {
            WorkflowInstanceId = started.WorkflowInstanceId,
        });
        Assert.That(instance.Instance.Status, Is.EqualTo("Completed"));
        Assert.That(transport.Calls.Any(call => call.ServiceName == "billing" && call.Path.EndsWith("invoices", StringComparison.Ordinal)), Is.True);
    }

    [Test]
    public void ACallerWithoutTheTaskRoleCannotCompleteTheTask()
    {
        Assert.ThrowsAsync(Is.InstanceOf<Exception>(), async () =>
        {
            using var scope = host!.Services.CreateScope();
            var runtime = scope.ServiceProvider.GetRequiredService<WorkflowRuntimeService>();
            var started = await runtime.StartWorkflowAsync(new StartWorkflowRequest
            {
                WorkflowName = "DocumentReview",
                Payload = new Dictionary<string, object?> { ["documentId"] = "DOC-8", ["title"] = "Leave policy", ["author"] = "author-1" },
            });

            await runtime.CompleteTaskAsync(new WorkflowTaskCompleteRequest
            {
                WorkflowTaskId = started.NextTasks.Single().WorkflowTaskId,
                ActorId = "author-1",
                ActorRoles = ["author"],
                Payload = new Dictionary<string, object?> { ["decision"] = "approve" },
            });
        });
    }
}

CanonicalExport.cs:

using StellaOps.Workflow.Abstractions;
using StellaOps.Workflow.Contracts;

namespace Examples.Workflow.Tutorials.Testing;

public static class CanonicalExport
{
    /// <summary>
    /// Writes the canonical JSON of a workflow to a file: the document to review, to validate, or to import
    /// into a workflow host.
    /// </summary>
    public static void WriteCanonicalJson<TStartRequest>(IDeclarativeWorkflow<TStartRequest> workflow, string path)
        where TStartRequest : class
    {
        var compilation = WorkflowCanonicalDefinitionCompiler.Compile(workflow);
        if (!compilation.Succeeded)
        {
            throw new InvalidOperationException(
                string.Join("; ", compilation.Diagnostics.Select(diagnostic => $"{diagnostic.Code}: {diagnostic.Message}")));
        }

        File.WriteAllText(path, WorkflowCanonicalJsonSerializer.Serialize(compilation.Definition!));
    }
}

FinishedInstanceCounter.cs:

using System.Collections.Concurrent;
using System.Diagnostics.Metrics;

namespace Examples.Workflow.Tutorials.Testing;

/// <summary>
/// Counts the finished workflow instances of this process from the engine's metrics: the counter
/// <c>workflow.execution.terminal</c> of the meter <c>StellaOps.Workflow</c>, labelled
/// <c>definition</c> and <c>outcome</c>. A test, a health probe or a dashboard of your own can read it
/// without an OpenTelemetry pipeline.
/// </summary>
public sealed class FinishedInstanceCounter : IDisposable
{
    private readonly MeterListener listener = new();
    private readonly ConcurrentDictionary<(string Definition, string Outcome), long> counts = new();

    public FinishedInstanceCounter()
    {
        listener.InstrumentPublished = (instrument, meterListener) =>
        {
            if (instrument.Meter.Name == "StellaOps.Workflow" && instrument.Name == "workflow.execution.terminal")
            {
                meterListener.EnableMeasurementEvents(instrument);
            }
        };
        listener.SetMeasurementEventCallback<long>((_, value, tags, _) =>
        {
            string definition = string.Empty, outcome = string.Empty;
            foreach (var tag in tags)
            {
                if (tag.Key == "definition") definition = tag.Value?.ToString() ?? string.Empty;
                if (tag.Key == "outcome") outcome = tag.Value?.ToString() ?? string.Empty;
            }

            counts.AddOrUpdate((definition, outcome), value, (_, total) => total + value);
        });
        listener.Start();
    }

    /// <summary>The instances of a workflow that finished with an outcome, for example <c>Completed</c>.</summary>
    public long Finished(string definition, string outcome = "Completed")
        => counts.TryGetValue((definition, outcome), out var count) ? count : 0;

    public void Dispose() => listener.Dispose();
}

Packages

StellaOps.Workflow (the engine with the PostgreSQL store), Microsoft.Extensions.Hosting, Testcontainers.PostgreSql, and your test framework (the examples use NUnit). Docker must run on the test machine; the runtime tests are skipped when it does not.

More cases and pitfalls: Testing.

Next: Lesson 10: Rendering.