DataFuse Framework

Real-World Use Cases

DataFuse shines when a single response needs data from multiple backend systems. Here are scenarios where it eliminates boilerplate and adds automatic parallelism.

public class ProductPageConfig : EntityConfiguration<ProductPage>
{
    public override IEnumerable<Mapping<ProductPage, IQueryResult>> GetSchema()
    {
        return CreateSchema.For<ProductPage>()
            .Map<ProductQuery, ProductTransform>(For.Paths("product"),
                p => p.Dependents
                    .Map<PricingApiQuery, PricingTransform>(For.Paths("product/pricing"))
                    .Map<ReviewsApiQuery, ReviewsTransform>(For.Paths("product/reviews")))
            .End();
    }
}

// Full customer 360
var customer = dataProvider.GetData(new CustomerRequest { CustomerId = 123 });

// Just profile + billing (skip tickets and orders)
var billing = dataProvider.GetData(new CustomerRequest
{
    CustomerId = 123,
    SchemaPaths = new[] { "customer", "customer/billing" }
});

Why DataFuse?

The Problem

In modern architectures, a single page or API response often needs data from multiple backend systems. The standard approach is manual orchestration code:

// Without DataFuse: 50+ lines per data combination, repeated everywhere
var customer = await GetCustomerFromDatabase(customerId);
var orders = await GetOrdersFromAPI(customerId);           // Sequential!
var billing = await GetBillingFromService(customerId);     // Sequential!
var tickets = await GetTicketsFromAPI(customerId);          // Sequential!

customer.Orders = orders;
customer.Billing = billing;
customer.Tickets = tickets;

// With DataFuse: declare once, use everywhere
var customer = dataProvider.GetData(new CustomerRequest { CustomerId = 123 });

Comparison with Alternatives

Quick Start (5 Minutes)

1. Install Packages

2. Define Your Entity

public class Product : IEntity
{
    public int ProductId { get; set; }
    public string Name { get; set; }
    public decimal Price { get; set; }
    public Category Category { get; set; }
    public Review[] Reviews { get; set; }
}

3. Create a Query

public class ProductQuery : SQLQuery<ProductResult>
{
    protected override Func<IDbConnection, Task<ProductResult>> GetQuery(
        IDataContext context, IQueryResult parentQueryResult)
    {
        var request = (ProductRequest)context.Request;
        return connection => connection.QueryFirstOrDefaultAsync<ProductResult>(
            "SELECT ProductId as Id, Name, Price FROM Products WHERE ProductId = @Id",
            new { Id = request.ProductId });
    }
}

public class ReviewsApiQuery : WebQuery<CollectionResult<ReviewResult>>
{
    public ReviewsApiQuery() : base("https://api.reviews.com/") { }

    protected override Func<Uri> GetQuery(IDataContext context, IQueryResult parentQueryResult)
    {
        var product = (ProductResult)parentQueryResult;
        return () => new Uri($"products/{product.Id}/reviews", UriKind.Relative);
    }
}

4. Create Transformers

public class ProductTransform : BaseTransformer<ProductResult, Product>
{
    public override void Transform(ProductResult queryResult, Product entity)
    {
        entity.ProductId = queryResult.Id;
        entity.Name = queryResult.Name;
        entity.Price = queryResult.Price;
    }
}

public class ReviewsTransform : BaseTransformer<CollectionResult<ReviewResult>, Product>
{
    public override void Transform(CollectionResult<ReviewResult> queryResult, Product entity)
    {
        entity.Reviews = queryResult?.Select(r => new Review
        {
            ReviewId = r.Id, Comment = r.Comment, Rating = r.Rating
        }).ToArray() ?? Array.Empty<Review>();
    }
}

5. Configure Schema

public class ProductConfiguration : EntityConfiguration<Product>
{
    public override IEnumerable<Mapping<Product, IQueryResult>> GetSchema()
    {
        return CreateSchema.For<Product>()
            .Map<ProductQuery, ProductTransform>(For.Paths("product"),
                product => product.Dependents
                    .Map<CategoryQuery, CategoryTransform>(For.Paths("product/category"))
                    .Map<ReviewsApiQuery, ReviewsTransform>(For.Paths("product/reviews")))
            .End();
    }
}

6. Register & Use

// Startup / DI registration
services.UseDataFuse()
    .WithEngine(c => new QueryEngine(sqlConfiguration))
    .WithEngine<DataFuse.Adapters.WebAPI.QueryEngine>()
    .WithPathMatcher(c => new XPathMatcher())
    .WithEntityConfiguration<Product>(c => new ProductConfiguration());

services.AddHttpClient();

// Usage
public class ProductService
{
    private readonly IDataProvider<Product> _dataProvider;

    public ProductService(IDataProvider<Product> dataProvider)
        => _dataProvider = dataProvider;

    public Product GetProduct(int productId)
        => _dataProvider.GetData(new ProductRequest { ProductId = productId });

    public Product GetProductWithReviews(int productId)
        => _dataProvider.GetData(new ProductRequest
        {
            ProductId = productId,
            SchemaPaths = new[] { "product", "product/reviews" }
        });
}

Core Concepts

Entities

Entities implement IEntity and define the complete object graph. Each property can be hydrated from a different data source:

public class Customer : IEntity
{
    public int CustomerId { get; set; }
    public string Name { get; set; }
    public Communication Communication { get; set; }  // From REST API
    public Address Address { get; set; }                // From SQL
    public Order[] Orders { get; set; }                 // From EF Core
}

Queries & Parent-Child Dependencies

Child queries receive their parent's result, enabling dependent fetching across data sources:

// Root: fetches from SQL
public class CustomerQuery : SQLQuery<CustomerResult> { /* uses request context */ }

// Child: uses parent CustomerResult to call API
public class OrdersApiQuery : WebQuery<CollectionResult<OrderResult>>
{
    protected override Func<Uri> GetQuery(IDataContext context, IQueryResult parentQueryResult)
    {
        var customer = (CustomerResult)parentQueryResult;
        return () => new Uri($"orders?customerId={customer.Id}", UriKind.Relative);
    }
}

Execution Flow

1. Root Query executes first
   |
   v  Result passed to children

2. Child Queries execute IN PARALLEL
   - OrdersQuery, CommunicationQuery, AddressQuery
   |
   v  Results passed to grandchildren

3. Grandchild Queries execute
   - OrderItemsQuery (uses OrderId from parent)

Transformers

Transformers map query results onto the entity. Each query-transformer pair handles one piece of the object graph:

public class OrdersTransform : BaseTransformer<CollectionResult<OrderResult>, Customer>
{
    public override void Transform(CollectionResult<OrderResult> queryResult, Customer entity)
    {
        entity.Orders = queryResult?.Select(o => new Order
        {
            OrderId = o.OrderId,
            OrderNumber = o.OrderNumber,
            OrderDate = o.OrderDate
        }).ToArray() ?? Array.Empty<Order>();
    }
}

Schema Configuration

The schema declares the full hierarchy of queries, transformers, and path mappings:

return CreateSchema.For<Customer>()
    .Map<CustomerQuery, CustomerTransform>(For.Paths("customer"),
        c => c.Dependents
            .Map<CommunicationQuery, CommTransform>(For.Paths("customer/communication"))
            .Map<OrdersQuery, OrdersTransform>(For.Paths("customer/orders"),
                o => o.Dependents
                    .Map<ItemsQuery, ItemsTransform>(For.Paths("customer/orders/order/items"))))
    .End();

Packages

Advanced Features

Selective Loading

Fetch only the parts of the object graph you need via SchemaPaths:

// Load everything
var full = dataProvider.GetData(new CustomerRequest { CustomerId = 123 });

// Load only customer + orders
var partial = dataProvider.GetData(new CustomerRequest
{
    CustomerId = 123,
    SchemaPaths = new[] { "customer", "customer/orders" }
});

Caching

Mark query results with [CacheResult] to make them available across queries and transformers:

[CacheResult]
public class CategoryResult : IQueryResult
{
    public int Id { get; set; }
    public string Name { get; set; }
}

// Access in a transformer
if (Context.Cache.TryGetValue("CategoryResult", out var cached))
{
    entity.CategoryName = ((CategoryResult)cached).Name;
}

Custom Data Source Adapters

Add support for any data source by implementing IQueryEngine:

public class MongoQueryEngine : IQueryEngine
{
    private readonly IMongoDatabase _db;
    public MongoQueryEngine(IMongoDatabase db) => _db = db;

    public bool CanExecute(IQuery query) => query is IMongoQuery;

    public async Task<IQueryResult> Execute(IQuery query)
    {
        var mongoQuery = (IMongoQuery)query;
        return await mongoQuery.Execute(_db);
    }
}

// Register alongside other engines
services.UseDataFuse()
    .WithEngine(c => new MongoQueryEngine(mongoDb))
    .WithEngine(c => new QueryEngine(sqlConfig));

Conditional Query Execution

Skip queries based on parent data by returning null:

protected override Func<Uri> GetQuery(IDataContext context, IQueryResult parentQueryResult)
{
    var customer = (CustomerResult)parentQueryResult;
    if (customer.CustomerType != "Premium")
        return null; // Query is skipped

    return () => new Uri($"premium/{customer.Id}", UriKind.Relative);
}

Path Matching

DataFuse supports XPath and JSONPath patterns for schema paths: