Barrel
FR GitHub

Query two databases as contexts

This project registers two databases, support tickets for two regions, as contexts and runs one BQL statement over both. The answer merges the rows in ORDER BY order and says which context each row came from, which contexts answered and what each answer covers. A third context points at a node that is down: the same query then returns a partial answer, with the reason. Use contexts when an agent must search data spread over several databases or nodes and needs to know what it did not see.

Prerequisites

  • Erlang/OTP 26 or later (this run used OTP 29) and rebar3.
  • A C compiler and CMake for the native dependencies (first compile takes a few minutes).

Run

./run.sh

run.sh compiles the project, deletes ./data, then runs contexts_bql:main/0 once.

What it shows

  • barrel_ctx:register/1: a card with a name and a location, local (a database on this node) or remote (a barrel_server endpoint and a database there).
  • barrel_ctx:query/1 with query and contexts: every context runs the same statement; the FROM name is ignored.
  • execution is succeeded, partial or failed; sources has one entry per context with its status, row count and bound; coverage counts them; summary says the same in one paragraph.
  • With offline => true remote contexts are skipped, not contacted.
  • A row query over contexts needs LIMIT: without it the request is refused before any context runs.

Notes

  • Not supported across contexts: SUBSCRIBE, OFFSET, UNNEST, continuations, and merging BM25 or hybrid scores (those come back grouped per context).
  • The same queries run over REST (POST /contexts/_query) and MCP (context_query) on barrel_server.

Expected output

Three answers and one refusal. The output of a real run is in expected.txt.

Download The folder on GitHub, with every file below.

Source

src/contexts_bql.erl
%% Two databases registered as contexts and queried together with
%% one BQL statement; a third context that cannot answer shows a
%% partial result.
-module(contexts_bql).
-export([main/0]).

-define(Q, <<"SELECT id, subject, minutes FROM c "
             "ORDER BY minutes DESC LIMIT 3">>).

main() ->
    try run() of
        ok -> halt(0)
    catch C:R:S ->
        io:format(standard_error, "~p:~p~n~p~n", [C, R, S]),
        halt(1)
    end.

run() ->
    {ok, _} = application:ensure_all_started(barrel),

    %% Two ordinary databases, one per region.
    {ok, Eu} = barrel_dbs:ensure(<<"tickets_eu">>, #{vectordb => none}),
    {ok, Us} = barrel_dbs:ensure(<<"tickets_us">>, #{vectordb => none}),
    _ = barrel:put_docs(Eu, [
        ticket(<<"t:1">>, <<"Invoice PDF missing">>, 42),
        ticket(<<"t:2">>, <<"SSO loop on Safari">>, 95),
        ticket(<<"t:3">>, <<"CSV export times out">>, 30)]),
    _ = barrel:put_docs(Us, [
        ticket(<<"t:7">>, <<"Password reset email late">>, 61),
        ticket(<<"t:8">>, <<"Webhook retries twice">>, 12)]),

    %% One card per context. A card names where the data lives.
    ok = add_context(<<"support/eu">>, local(<<"tickets_eu">>)),
    ok = add_context(<<"support/us">>, local(<<"tickets_us">>)),
    %% A remote node that is down (nothing listens on port 1).
    ok = add_context(<<"support/apac">>,
                  #{<<"kind">> => <<"remote">>,
                    <<"endpoint">> => <<"http://127.0.0.1:1">>,
                    <<"db">> => <<"tickets_apac">>}),

    io:format("== eu and us~n"),
    report(barrel_ctx:query(#{
        query => ?Q,
        contexts => [<<"support/eu">>, <<"support/us">>]})),

    io:format("~n== eu, us and apac (apac is down)~n"),
    report(barrel_ctx:query(#{
        query => ?Q,
        contexts => [<<"support/eu">>, <<"support/us">>,
                     <<"support/apac">>]})),

    io:format("~n== the same, offline~n"),
    report(barrel_ctx:query(#{
        query => ?Q, offline => true,
        contexts => [<<"support/eu">>, <<"support/us">>,
                     <<"support/apac">>]})),

    io:format("~n== a row query without LIMIT~n"),
    io:format("~p~n", [barrel_ctx:query(#{
        query => <<"SELECT id FROM c">>,
        contexts => [<<"support/eu">>]})]),
    ok.

ticket(Id, Subject, Minutes) ->
    #{<<"id">> => Id, <<"subject">> => Subject,
      <<"minutes">> => Minutes}.

local(Db) -> #{<<"kind">> => <<"local">>, <<"db">> => Db}.

add_context(Name, Location) ->
    {ok, _Card} = barrel_ctx:register(
        #{<<"name">> => Name, <<"locations">> => [Location]}),
    ok.

report({ok, R}) ->
    io:format("execution: ~p, merge: ~p~n",
              [maps:get(execution, R), maps:get(merge, R)]),
    io:format("rows:~n"),
    [io:format("  ~-12s ~-5s ~3b  ~s~n",
               [maps:get(<<"_ctx_name">>, Row), maps:get(<<"id">>, Row),
                maps:get(<<"minutes">>, Row),
                maps:get(<<"subject">>, Row)])
     || Row <- maps:get(rows, R)],
    io:format("sources:~n"),
    [source(S) || S <- maps:get(sources, R)],
    C = maps:get(coverage, R),
    io:format("coverage: requested ~b, answered ~b, failed ~b, "
              "skipped ~b~n",
              [maps:get(K, C) || K <- [requested, answered, failed,
                                       skipped]]),
    io:format("summary:~n"),
    [io:format("  ~s~n", [L]) || L <- wrap(maps:get(summary, R), 62)],
    ok.

source(#{name := N, status := St} = S) ->
    case S of
        #{error := #{reason := Why}} ->
            io:format("  ~-12s ~p: ~p~n", [N, St, Why]);
        #{rows := Rows, bound := Bound} ->
            io:format("  ~-12s ~p, ~b rows, ~p~n", [N, St, Rows, Bound])
    end.

%% Word-wrap a binary to lines of at most Width characters.
wrap(Text, Width) ->
    Words = string:lexemes(Text, " "),
    {Lines, Last} = lists:foldl(
        fun(W, {Acc, <<>>}) -> {Acc, W};
           (W, {Acc, Cur}) ->
                case string:length(Cur) + 1 + string:length(W) of
                    L when L > Width -> {[Cur | Acc], W};
                    _ -> {Acc, <<Cur/binary, " ", W/binary>>}
                end
        end, {[], <<>>}, Words),
    lists:reverse([Last | Lines]).
rebar.config
{erl_opts, [debug_info]}.
{deps, [{barrel, "~> 1.12"}]}.
run.sh
#!/bin/sh
# Build, then run main/0 once on a fresh data directory.
set -e
cd "$(dirname "$0")"
rebar3 compile >&2
rm -rf data
exec erl -noshell -config config/sys \
  -pa _build/default/lib/*/ebin _build/default/checkouts/*/ebin \
  -eval 'contexts_bql:main()'

Expected output

The expected.txt of the example, from a real run. Revisions, ids, tokens and timestamps differ on every run; scripts/verify-examples.sh masks them before it compares.

expected.txt
== eu and us
execution: succeeded, merge: ordered
rows:
  support/eu   t:2    95  SSO loop on Safari
  support/us   t:7    61  Password reset email late
  support/eu   t:1    42  Invoice PDF missing
sources:
  support/eu   ok, 3 rows, limit_reached
  support/us   ok, 2 rows, exhausted
coverage: requested 2, answered 2, failed 0, skipped 0
summary:
  All 2 contexts answered (support/eu, support/us). 3 rows are
  merged across contexts in ORDER BY order. support/eu filled
  the LIMIT or k; more matches may exist.

== eu, us and apac (apac is down)
execution: partial, merge: ordered
rows:
  support/eu   t:2    95  SSO loop on Safari
  support/us   t:7    61  Password reset email late
  support/eu   t:1    42  Invoice PDF missing
sources:
  support/eu   ok, 3 rows, limit_reached
  support/us   ok, 2 rows, exhausted
  support/apac unreachable: econnrefused
coverage: requested 3, answered 2, failed 1, skipped 0
summary:
  2 of 3 contexts answered; rows from the others are not
  included. support/apac: the server did not accept the
  connection (econnrefused). 3 rows are merged across contexts
  in ORDER BY order. support/eu filled the LIMIT or k; more
  matches may exist.

== the same, offline
execution: partial, merge: ordered
rows:
  support/eu   t:2    95  SSO loop on Safari
  support/us   t:7    61  Password reset email late
  support/eu   t:1    42  Invoice PDF missing
sources:
  support/eu   ok, 3 rows, limit_reached
  support/us   ok, 2 rows, exhausted
  support/apac skipped_offline: no_local_copy
coverage: requested 3, answered 2, failed 0, skipped 1
summary:
  2 of 3 contexts answered; rows from the others are not
  included. support/apac: skipped: offline and no local copy. 3
  rows are merged across contexts in ORDER BY order. support/eu
  filled the LIMIT or k; more matches may exist.

== a row query without LIMIT
{error,{limit_required,#{max_limit => 1000}}}

Up and down to moveEnter to open