nicolas
/
triangular_arbitrage_bot
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: per-slot fill channels with orderId fallback routing; fix reconnect double-subscribe 

Architecture: each executor slot owns its own fill_channel_t. Executor encodes
slot_index in client_oid bytes 1-2 ("c{slot:02x}{ts:08x}{leg:04x}"). Hot thread
decodes slot from client_oid to route match events. For terminal events without
clientOid, falls back to orderId->slot_index mapping registered after each
successful rest_order_place.

Reconnect fixes:
- Remove redundant ws_client_subscribe after reconnect (already done in
  ws_client_connect) - caused "exceed max subscription count" 509 errors
- Only advance last_activity_ms on successful reconnect, so failed attempts
  retry every 100ms instead of every 5s
- Private channel subscriptions gated to connection 0 only to prevent
  duplicate fill events from KuCoin broadcasting to all connections
This commit is contained in:
nicolas 2026-06-02 18:29:30 -03:00
parent d569063c75
commit b0056f4b6b
7 changed files with 131 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/events.c
View File

@ -225,16 +225,16 @@ void *event_executor_thread(void *arg) {
executor_slot_t *slot = ta->slot;
executor_thread_t *exec = executor_thread_create(loops->ws_client->cfg,
loops->ws_client->fill_ch,
loops->ws_client,
loops->executor_shared,
slot);
slot,
ta->slot_index);
if (!exec) {
log_write("[EXEC] Failed to create executor\n");
return NULL;
}
int fill_wake_fd = fill_channel_wake_fd(loops->ws_client->fill_ch);
int fill_wake_fd = fill_channel_wake_fd(slot->fill_ch);
struct pollfd pfds[2];
memset(pfds, 0, sizeof(pfds));
pfds[0].fd = slot->eventfd;

41
src/executor.c
View File

@ -18,6 +18,7 @@ struct executor_thread_s {
ws_client_t *ws;
executor_shared_t *shared;
executor_slot_t *slot;
int slot_index;
};
/* ── Reporting ── */
@ -72,17 +73,17 @@ static double apply_increment_floor(double vol, double inc) {
/* ── Core execution loop ── */
executor_thread_t *executor_thread_create(const config_t *cfg,
fill_channel_t *fill_ch,
ws_client_t *ws,
executor_shared_t *shared,
executor_slot_t *slot) {
(void)fill_ch;
executor_slot_t *slot,
int slot_index) {
executor_thread_t *et = calloc(1, sizeof(*et));
if (!et) return NULL;
et->cfg = cfg;
et->ws = ws;
et->shared = shared;
et->slot = slot;
et->slot_index = slot_index;
et->rest = rest_conn_new();
if (et->rest) {
rest_conn_set_auth(et->rest,
@ -107,7 +108,6 @@ bool executor_keepalive(executor_thread_t *et) {
void executor_execute_triangle(executor_thread_t *et,
signal_entry_t *sig) {
fill_channel_t *fill_ch = et->ws->fill_ch;
executor_shared_t *sh = et->shared;
/* ── Stale book check: skip if book_ts is not newer than last execution ── */
@ -208,9 +208,10 @@ void executor_execute_triangle(executor_thread_t *et,
is_buy ? (sl->funds_increment > 0 ? sl->funds_increment : sl->quote_increment) : sl->base_increment);
}
/* Build a client OID */
/* Build a client OID with slot index for fill routing */
char client_oid[64];
snprintf(client_oid, sizeof(client_oid), "c%08x%04x",
snprintf(client_oid, sizeof(client_oid), "c%02x%08x%04x",
(unsigned)et->slot_index,
(unsigned)((uint64_t)now_realtime_ms() & 0xFFFFFFFF),
(unsigned)leg);
@ -268,12 +269,18 @@ void executor_execute_triangle(executor_thread_t *et,
break;
}
/* Register orderId for fill event routing (hot thread uses this to
route terminal events without clientOid to the right slot). */
if (ok && order_id[0]) {
ws_client_register_order_slot(et->ws, order_id, et->slot_index);
}
/* ── Wait for fill (live only) ── */
double total_size = 0, total_funds = 0, avg_price = 0, total_fee = 0;
if (sig->live) {
fill_result_t fr = {0};
bool filled = fill_channel_await(fill_ch, client_oid,
bool filled = fill_channel_await(et->slot->fill_ch, client_oid,
FILL_TIMEOUT_MS, &fr);
if (!filled) {
error_str = fr.match_count > 0 ? "partial_fill" : "fill_timeout";
@ -474,6 +481,22 @@ void executor_execute_triangle(executor_thread_t *et,
if (fills_pos >= (int)sizeof(fills_str) - 1) break;
}
/* ── Build books string: top-of-book at signal time ── */
char books_str[512] = "";
int books_pos = 0;
for (int leg = 0; leg < 3; leg++) {
const signal_book_t *bk = &sig->books[leg];
if (bk->symbol[0] == '\0') continue;
books_pos += snprintf(books_str + books_pos, sizeof(books_str) - books_pos,
"%s%s: bid=%.8g@%.4g ask=%.8g@%.4g",
leg > 0 ? ", " : "", bk->symbol,
bk->bid_count > 0 ? bk->bids[0].price : 0.0,
bk->bid_count > 0 ? bk->bids[0].size : 0.0,
bk->ask_count > 0 ? bk->asks[0].price : 0.0,
bk->ask_count > 0 ? bk->asks[0].size : 0.0);
if (books_pos >= (int)sizeof(books_str) - 1) break;
}
/* ── Emit report (single line) ── */
format_ts(ts_buf, sizeof(ts_buf));
const char *status = success ? "FILLED" : "FAILED";
@ -484,11 +507,11 @@ void executor_execute_triangle(executor_thread_t *et,
"%s %s | corr=%s | triangle=['%s'] | "
"predicted_bps=%.2f | effective_bps=%s | "
"book_ts=%lld | profit=%.4f | timings=[%s] | "
"fills=[%s]%s%s\n",
"fills=[%s] | books=[%s]%s%s\n",
ts_buf, status, corr_id, sig->triangle_key,
sig->predicted_bps, bps_str,
(long long)sig->book_ts_ms, profit, timings_str,
fills_str,
fills_str, books_str,
error_str[0] ? " | error=" : "",
error_str[0] ? error_str : "");
/* Release isolation slot */

4
src/executor.h
View File

@ -28,10 +28,10 @@ typedef struct executor_thread_s executor_thread_t;
/* Create an executor thread (one per concurrent slot). */
executor_thread_t *executor_thread_create(const config_t *cfg,
fill_channel_t *fill_ch,
ws_client_t *ws,
executor_shared_t *shared,
executor_slot_t *slot);
executor_slot_t *slot,
int slot_index);
/* Execute a single triangle signal (blocking, called from executor thread). */
void executor_execute_triangle(executor_thread_t *et,

15
src/main.c
View File

@ -116,6 +116,13 @@ int main(int argc, char *argv[]) {
return 1;
}
slots[i].fill_ch = fill_channel_create();
if (!slots[i].fill_ch) {
log_write("[MAIN] Failed to create slot fill channel\n");
for (int j = 0; j < i; j++) { close(slots[j].eventfd); fill_channel_destroy(slots[j].fill_ch); }
free(slots);
return 1;
}
}
log_write("[MAIN] Created %d executor slots\n", n_slots);
@ -124,7 +131,7 @@ int main(int argc, char *argv[]) {
cfg.kcs_discount_active);
ws_client_t ws_client;
if (ws_client_init(&ws_client, &cfg, &symbols, books, &evaluator) != 0) {
if (ws_client_init(&ws_client, &cfg, &symbols, books, &evaluator, slots, n_slots) != 0) {
log_write("[MAIN] Failed to init WS client\n");
triangle_set_free(&triangles);
@ -218,11 +225,8 @@ int main(int argc, char *argv[]) {
int fl = fcntl(conn->fd, F_GETFL);
fcntl(conn->fd, F_SETFL, fl | O_NONBLOCK);
event_loops_add_fd(&events.hot_epoll, conn->fd, FD_TYPE_WS, i, NULL, EPOLLIN);
if (conn->symbol_count > 0) {
ws_client_subscribe(&ws_client, i,
conn->symbol_indices, conn->symbol_count);
}
}
if (conn->state == WS_STATE_CONNECTED)
conn->last_activity_ms = now;
}
}
@ -248,6 +252,7 @@ int main(int argc, char *argv[]) {
ws_client_destroy(&ws_client);
for (int i = 0; i < n_slots; i++) {
close(slots[i].eventfd);
fill_channel_destroy(slots[i].fill_ch);
}
free(slots);
triangle_set_free(&triangles);

2
src/slot.h
View File

@ -4,6 +4,7 @@
#include <stdint.h>
#include <stdatomic.h>
#include "triangle.h"
#include "fill_handler.h"
#define EXECUTOR_SLOT_FREE 0
#define EXECUTOR_SLOT_CLAIMED 1
@ -14,6 +15,7 @@ typedef struct {
_Atomic int state;
signal_entry_t signal;
int eventfd;
fill_channel_t *fill_ch;
} executor_slot_t;
bool slot_deliver(executor_slot_t *slots, int n_slots, const signal_entry_t *sig);

72
src/ws_client.c
View File

@ -156,13 +156,16 @@ static int setup_tls(ws_connection_t *conn) {
int ws_client_init(ws_client_t *client, const config_t *cfg,
symbol_table_t *symbols, order_book_t *books,
evaluator_t *evaluator) {
evaluator_t *evaluator,
executor_slot_t *slots, int n_slots) {
memset(client, 0, sizeof(*client));
client->cfg = cfg;
client->symbols = symbols;
client->books = books;
client->evaluator = evaluator;
client->running = true;
client->slots = slots;
client->n_slots = n_slots;
SSL_CTX *shared_ctx = create_ssl_ctx();
if (!shared_ctx) {
@ -188,6 +191,7 @@ int ws_client_init(ws_client_t *client, const config_t *cfg,
}
pthread_mutex_init(&client->balance_lock, NULL);
pthread_mutex_init(&client->order_slots_lock, NULL);
client->balance_wake_fd = eventfd(0, EFD_NONBLOCK);
if (client->balance_wake_fd < 0) {
log_write("[WS] Failed to create balance wake eventfd\n");
@ -196,6 +200,22 @@ int ws_client_init(ws_client_t *client, const config_t *cfg,
return 0;
}
void ws_client_register_order_slot(ws_client_t *client,
const char *order_id, int slot_index) {
if (!order_id || !order_id[0]) return;
pthread_mutex_lock(&client->order_slots_lock);
for (int i = 0; i < MAX_ORDER_SLOT_ENTRIES; i++) {
if (!atomic_load_explicit(&client->order_slots[i].active, memory_order_relaxed)) {
strncpy(client->order_slots[i].order_id, order_id, sizeof(client->order_slots[i].order_id) - 1);
client->order_slots[i].slot_index = slot_index;
atomic_store_explicit(&client->order_slots[i].active, 1, memory_order_release);
pthread_mutex_unlock(&client->order_slots_lock);
return;
}
}
pthread_mutex_unlock(&client->order_slots_lock);
}
void ws_client_destroy(ws_client_t *client) {
client->running = false;
SSL_CTX *ctx = NULL;
@ -383,17 +403,16 @@ int ws_client_connect(ws_client_t *client, uint32_t conn_idx) {
ws_client_subscribe(client, conn_idx, conn->symbol_indices, conn->symbol_count);
}
/* Subscribe to private channels */
{
/* Subscribe to private channels — only on connection 0 to avoid
duplicate delivery of order/balance events across connections */
if (conn_idx == 0) {
char msg[256];
snprintf(msg, sizeof(msg),
"{\"type\":\"subscribe\",\"topic\":\"/spotMarket/tradeOrdersV2\","
"\"response\":true,\"privateChannel\":\"true\"}");
ws_send_text(conn, msg);
log_write("[WS] Subscribed to tradeOrdersV2\n");
}
{
char msg[256];
snprintf(msg, sizeof(msg),
"{\"type\":\"subscribe\",\"topic\":\"/account/balance\","
"\"response\":true,\"privateChannel\":\"true\"}");
@ -777,6 +796,39 @@ int16_t ws_client_process_frame(ws_client_t *client, uint32_t conn_idx) {
cJSON_IsString(status) &&
strcmp(status->valuestring, "done") == 0);
/* Route to the correct slot's fill channel.
First try clientOid (slot encoded in bytes 1-2),
then fall back to orderId lookup. */
fill_channel_t *target_ch = client->fill_ch;
const char *coid = oid->valuestring;
if (coid[0] == 'c' && coid[1] && coid[2]) {
unsigned long slot_idx = 0;
for (int h = 1; h <= 2; h++) {
char c = coid[h];
slot_idx *= 16;
if (c >= '0' && c <= '9') slot_idx += (unsigned long)(c - '0');
else if (c >= 'a' && c <= 'f') slot_idx += (unsigned long)(c - 'a' + 10);
else if (c >= 'A' && c <= 'F') slot_idx += (unsigned long)(c - 'A' + 10);
}
if (slot_idx < (unsigned long)client->n_slots && client->slots)
target_ch = client->slots[slot_idx].fill_ch;
}
/* If clientOid routing failed, try orderId mapping */
if (target_ch == client->fill_ch && order_id && cJSON_IsString(order_id)) {
const char *oid_str = order_id->valuestring;
pthread_mutex_lock(&client->order_slots_lock);
for (int i = 0; i < MAX_ORDER_SLOT_ENTRIES; i++) {
if (atomic_load_explicit(&client->order_slots[i].active, memory_order_acquire) &&
strcmp(client->order_slots[i].order_id, oid_str) == 0) {
int si = client->order_slots[i].slot_index;
if (si >= 0 && si < client->n_slots && client->slots)
target_ch = client->slots[si].fill_ch;
break;
}
}
pthread_mutex_unlock(&client->order_slots_lock);
}
if (is_match) {
cJSON *ms = cJSON_GetObjectItem(data, "matchSize");
cJSON *mp = cJSON_GetObjectItem(data, "matchPrice");
@ -787,16 +839,12 @@ int16_t ws_client_process_frame(ws_client_t *client, uint32_t conn_idx) {
cJSON_IsNumber(mp) ? mp->valuedouble : 0;
fe.match_fee = cJSON_IsString(mf) ? atof(mf->valuestring) :
cJSON_IsNumber(mf) ? mf->valuedouble : 0;
if (!fill_channel_push(client->fill_ch, &fe) &&
if (!fill_channel_push(target_ch, &fe) &&
++client->fill_drop_warn <= 3)
log_write("[WS] Fill ring full, dropping match\n");
} else if (is_terminal) {
/* Terminal event: push a marker that stops accumulation.
A preceding "match" event already communicated the data.
If no matches arrived, this is a cancelled+empty order.
We push a terminal marker to unblock the waiter. */
fe.is_terminal = true;
if (!fill_channel_push(client->fill_ch, &fe) &&
if (!fill_channel_push(target_ch, &fe) &&
++client->fill_drop_warn <= 3)
log_write("[WS] Fill ring full, dropping terminal\n");
}

22
src/ws_client.h
View File

@ -11,6 +11,7 @@
#include "hash.h"
#include "evaluate.h"
#include "fill_handler.h"
#include "slot.h"
#define MAX_BALANCE_ENTRIES 64
@ -71,6 +72,14 @@ typedef struct {
uint32_t symbol_count; /* number of subscribed symbols */
} ws_connection_t;
typedef struct {
char order_id[32];
int slot_index;
_Atomic int active;
} order_slot_entry_t;
#define MAX_ORDER_SLOT_ENTRIES 64
/* Top-level WebSocket client managing multiple connections */
typedef struct {
ws_connection_t connections[WS_MAX_CONNECTIONS]; /* fixed-size connection pool */
@ -80,18 +89,23 @@ typedef struct {
order_book_t *books; /* pointer to shared order books */
evaluator_t *evaluator; /* pointer to evaluator */
bool running; /* false signals client to stop */
executor_slot_t *slots; /* per-slot state for fill routing */
int n_slots; /* number of executor slots */
fill_channel_t *fill_ch; /* fill event channel (hot→executor) */
int fill_drop_warn; /* rate-limited fill drop warning counter */
balance_entry_t balance_cache[MAX_BALANCE_ENTRIES]; /* latest available balances from WS */
int balance_count;
pthread_mutex_t balance_lock;
int balance_wake_fd; /* eventfd: written on every balance update */
order_slot_entry_t order_slots[MAX_ORDER_SLOT_ENTRIES]; /* orderId→slot_index mapping */
pthread_mutex_t order_slots_lock;
} ws_client_t;
/* Initialise a WebSocket client with config, symbol table, books, and evaluator */
/* Initialise a WebSocket client with config, symbol table, books, evaluator, and slots */
int ws_client_init(ws_client_t *client, const config_t *cfg,
symbol_table_t *symbols, order_book_t *books,
evaluator_t *evaluator);
evaluator_t *evaluator,
executor_slot_t *slots, int n_slots);
/* Destroy WebSocket client and close all connections */
void ws_client_destroy(ws_client_t *client);
/* Initiate a WebSocket connection (non-blocking) */
@ -128,4 +142,8 @@ bool ws_client_await_balance(ws_client_t *client, const char *currency,
Returns 0 if currency is not in cache. */
double ws_client_latest_balance(ws_client_t *client, const char *currency);
/* Register orderId→slot_index mapping for fill event routing (executor thread). */
void ws_client_register_order_slot(ws_client_t *client,
const char *order_id, int slot_index);
#endif