From e1eb3afacc9287881a18d68dd5ac9ab7095db905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=A7=CE=B3=CF=86=CF=84=20Kompanion?= Date: Sun, 19 Oct 2025 08:32:43 +0200 Subject: [PATCH] middleware: add C++ orchestrator + API docs; cli: add MCP server mode and DB navigation; tests: add orchestrator QtTest; docs: add CLI usage; db: move SQL from sql/ to db/sql; tools: remove extra python embedding stub - Add src/middleware/orchestrator.{h,cpp} with simple journal.from_prompt task loop and Ollama-backed model provider. - Extend src/cli/KompanionApp.cpp with: - MCP server mode (--mcp-serve/--mcp-address) - DB helpers (--db-namespaces, --db-items, --db-search, --embedding-file) - Add tests/test_orchestrator.cpp and link Qt6::Network; keep tests passing. - Move legacy SQL from sql/pg/001_init.sql to db/sql/pg/001_init.sql for consistency. - Add docs/cli-and-schema-navigation.md with examples for server, tools, and schema navigation. - Remove tools/pg_search.py (redundant embedding stub). --- {sql => db/sql}/pg/001_init.sql | 1 + docs/cli-and-schema-navigation.md | 50 ++++++++ src/CMakeLists.txt | 5 +- src/cli/KompanionApp.cpp | 185 +++++++++++++++++++++++++++++- src/cli/py_embedder.py | 58 ++++++++++ src/middleware/orchestrator.cpp | 172 +++++++++++++++++++++++++++ src/middleware/orchestrator.h | 91 +++++++++++++++ tests/CMakeLists.txt | 6 + tests/test_orchestrator.cpp | 51 ++++++++ tools/pg_search.py | 48 -------- 10 files changed, 616 insertions(+), 51 deletions(-) rename {sql => db/sql}/pg/001_init.sql (99%) create mode 100644 docs/cli-and-schema-navigation.md create mode 100644 src/cli/py_embedder.py create mode 100644 src/middleware/orchestrator.cpp create mode 100644 src/middleware/orchestrator.h create mode 100644 tests/test_orchestrator.cpp delete mode 100644 tools/pg_search.py diff --git a/sql/pg/001_init.sql b/db/sql/pg/001_init.sql similarity index 99% rename from sql/pg/001_init.sql rename to db/sql/pg/001_init.sql index a14c7d9..63e1f77 100644 --- a/sql/pg/001_init.sql +++ b/db/sql/pg/001_init.sql @@ -84,3 +84,4 @@ DROP TRIGGER IF EXISTS trg_bump_revision ON memory_items; CREATE TRIGGER trg_bump_revision BEFORE UPDATE ON memory_items FOR EACH ROW EXECUTE FUNCTION bump_revision(); + diff --git a/docs/cli-and-schema-navigation.md b/docs/cli-and-schema-navigation.md new file mode 100644 index 0000000..b7f67c9 --- /dev/null +++ b/docs/cli-and-schema-navigation.md @@ -0,0 +1,50 @@ +Kompanion CLI and Schema Navigation + +This guide shows how to use the `kompanion` CLI to: +- Configure the database and apply init SQL +- Call MCP tools directly +- Run an MCP server (stdio or network) from the CLI +- Inspect and query the Postgres schema + +Prerequisites +- Build: `cmake -S . -B build && cmake --build build -j` +- Optional: set `PG_DSN` (e.g., `postgresql://kompanion:komup@localhost:5432/kompanion`) + +Initialization +- Run wizard and apply DB schema: `kompanion --init` + - Writes `~/.config/kompanion/kompanionrc` (or KConfig). Also sets `PG_DSN` for the session. + +MCP Tool Usage +- List tools: `kompanion --list` +- Single call with inline JSON: `kompanion kom.memory.v1.search_memory -r '{"namespace":"dev_knowledge","query":{"text":"embedding model","k":5}}'` +- Read request from stdin: `echo '{"namespace":"dev_knowledge","content":"hello","key":"note"}' | kompanion kom.memory.v1.save_context -i` +- Interactive loop: `kompanion -I kom.memory.v1.search_memory` then type `!prompt quick brown fox` + +Run MCP Server from CLI +- Stdio backend (default): `kompanion --mcp-serve` +- Explicit backend: `kompanion --mcp-serve stdio` +- Network backend address (if available): `kompanion --mcp-serve ws --mcp-address 127.0.0.1:8000` + +Database Navigation +Note: These helpers expect a reachable Postgres (`PG_DSN` set). If missing, the CLI falls back to an in‑memory stub for tool calls, but DB navigation requires Postgres. + +- List namespaces: `kompanion --db-namespaces` + - Output: `nameuuid` +- List recent items in a namespace: `kompanion --db-items --ns dev_knowledge [--limit 20]` + - Output: `item_idkeycontent_snippettags` +- Hybrid search within a namespace: + - Text-only: `kompanion --db-search --ns dev_knowledge --text "pgvector index" --limit 5` + - With embedding vector from file: `kompanion --db-search --ns dev_knowledge --embedding-file /path/vec.json --limit 5` + - `vec.json` must be a JSON array of numbers representing the embedding. + +Schema Guide (Postgres) +- Tables: `namespaces`, `memory_items`, `memory_chunks`, `embeddings`, `auth_secrets` +- Key indexes: + - `memory_items(namespace_id, key)` (unique when `key` not null) + - `memory_chunks.content_tsv` GIN (full‑text) + - `embeddings.vector` IVFFLAT with `vector_cosine_ops` (per‑model partial index) + +Tips +- For quick trials without Postgres, tool calls work in stub mode (in‑memory DAL). To exercise vector search and FTS, run the DB init scripts via `kompanion --init`. +- Use `kompanion --verbose` to echo JSON requests/responses. + diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a4193d9..558cabc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -5,6 +5,7 @@ add_library(kompanion_mw SHARED middleware/libkiexecutor.cpp middleware/regexregistry.cpp middleware/guardrailspolicy.cpp + middleware/orchestrator.cpp ) find_package(Qt6 REQUIRED COMPONENTS Core DBus) @@ -30,7 +31,7 @@ target_include_directories(kompanion_mw PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/middl target_sources(kompanion_mw PRIVATE ${KOMPANION_DBUS_ADAPTOR_SRCS} ${KOMPANION_DBUS_INTERFACE_SRCS}) -target_link_libraries(kompanion_mw PRIVATE Qt6::Core Qt6::DBus) +target_link_libraries(kompanion_mw PRIVATE Qt6::Core Qt6::DBus Qt6::Network) target_compile_definitions(kompanion_mw PRIVATE KOMPANION_MW_LIBRARY) -# Example executable wiring GUI/controller/executor together could be added later. \ No newline at end of file +# Example executable wiring GUI/controller/executor together could be added later. diff --git a/src/cli/KompanionApp.cpp b/src/cli/KompanionApp.cpp index bd3541c..7c60b0f 100644 --- a/src/cli/KompanionApp.cpp +++ b/src/cli/KompanionApp.cpp @@ -16,6 +16,8 @@ #include #include #include +#include +#include #ifdef HAVE_KCONFIG #include @@ -38,6 +40,7 @@ #include "mcp/KompanionQtServer.hpp" #include "mcp/RegisterTools.hpp" #include "dal/PgDal.hpp" +#include "mcp/KomMcpServer.hpp" namespace { @@ -580,6 +583,19 @@ bool runInitializationWizard(QTextStream& in, return false; } +bool connectDalFromEnv(ki::PgDal& dal, QTextStream& out) { + const char* envDsn = std::getenv("PG_DSN"); + if (!envDsn || !*envDsn) { + out << "PG_DSN not set; using in-memory DAL stub.\n"; + return dal.connect("stub://memory"); + } + if (!dal.connect(envDsn)) { + out << "Failed to connect to database; falling back to stub.\n"; + return dal.connect("stub://memory"); + } + return true; +} + int runInteractiveSession(KomMcpServer& server, const std::string& toolName, bool verbose) { @@ -669,6 +685,109 @@ void printToolList(const KomMcpServer& server) { out.flush(); } +int runMcpServer(QString backend, QString address, QTextStream& qerr) { + KomMcpServer logic; + register_default_tools(logic); + + ki::PgDal dal; + QTextStream qout(stdout); + connectDalFromEnv(dal, qout); + + const QStringList availableBackends = QMcpServer::backends(); + if (availableBackends.isEmpty()) { + qerr << "[kompanion] No MCP server backends detected in plugin search path.\n"; + } else if (!availableBackends.contains(backend)) { + qerr << "[kompanion] Backend '" << backend << "' not available. Known: " + << availableBackends.join('/') << "\n"; + return 1; + } + + KompanionQtServer server(backend, &logic, &dal); + if (backend == QStringLiteral("stdio")) server.start(); else server.start(address); + QCoreApplication::instance()->exec(); + return 0; +} + +// ---------- DB helpers (CLI) ---------- +bool dbListNamespaces(QTextStream& out) { + const char* dsn = std::getenv("PG_DSN"); + if (!dsn || !*dsn) { out << "PG_DSN not set.\n"; return false; } + QString err; + if (!testConnection(dsn, &err)) { out << "Connection failed: " << err << "\n"; return false; } + + const QString connName = QStringLiteral("kompanion_db_%1").arg(QRandomGenerator::global()->generate64(), 0, 16); + QSqlDatabase db = QSqlDatabase::addDatabase(QStringLiteral("QPSQL"), connName); + const auto cfg = configFromDsn(std::optional(dsn)); + db.setDatabaseName(cfg.dbname); db.setUserName(cfg.user); db.setPassword(cfg.password); + db.setHostName(cfg.useSocket ? cfg.socketPath : cfg.host); + bool ok=false; const int portValue = cfg.port.toInt(&ok); if (ok && portValue>0) db.setPort(portValue); + if (!db.open()) { out << "Open failed: " << db.lastError().text() << "\n"; QSqlDatabase::removeDatabase(connName); return false; } + QSqlQuery q(db); + if (!q.exec(QStringLiteral("SELECT id::text, name FROM namespaces ORDER BY name"))) { + out << q.lastError().text() << "\n"; db.close(); QSqlDatabase::removeDatabase(connName); return false; } + while (q.next()) { + out << q.value(1).toString() << "\t" << q.value(0).toString() << "\n"; + } + db.close(); QSqlDatabase::removeDatabase(connName); + return true; +} + +bool dbListItems(const QString& nsName, int limit, QTextStream& out) { + const char* dsn = std::getenv("PG_DSN"); + if (!dsn || !*dsn) { out << "PG_DSN not set.\n"; return false; } + const QString connName = QStringLiteral("kompanion_db_%1").arg(QRandomGenerator::global()->generate64(), 0, 16); + QSqlDatabase db = QSqlDatabase::addDatabase(QStringLiteral("QPSQL"), connName); + const auto cfg = configFromDsn(std::optional(dsn)); + db.setDatabaseName(cfg.dbname); db.setUserName(cfg.user); db.setPassword(cfg.password); + db.setHostName(cfg.useSocket ? cfg.socketPath : cfg.host); + bool ok=false; const int portValue = cfg.port.toInt(&ok); if (ok && portValue>0) db.setPort(portValue); + if (!db.open()) { out << "Open failed: " << db.lastError().text() << "\n"; QSqlDatabase::removeDatabase(connName); return false; } + QSqlQuery q(db); + q.prepare(QStringLiteral( + "SELECT i.id::text, COALESCE(i.key,''), LEFT(i.content, 120), array_to_string(i.tags, ',') " + "FROM memory_items i JOIN namespaces n ON n.id = i.namespace_id " + "WHERE n.name = :name ORDER BY i.created_at DESC LIMIT :lim")); + q.bindValue(":name", nsName); q.bindValue(":lim", limit); + if (!q.exec()) { out << q.lastError().text() << "\n"; db.close(); QSqlDatabase::removeDatabase(connName); return false; } + while (q.next()) { + out << q.value(0).toString() << '\t' << q.value(1).toString() << '\t' << q.value(2).toString().replace('\n',' ') << '\t' << q.value(3).toString() << "\n"; + } + db.close(); QSqlDatabase::removeDatabase(connName); + return true; +} + +bool dbSearch(const QString& nsName, const QString& text, const QString& embeddingFile, int k, QTextStream& out) { + ki::PgDal dal; connectDalFromEnv(dal, out); + auto ns = dal.findNamespace(nsName.toStdString()); + if (!ns) { out << "namespace not found\n"; return false; } + std::vector vec; + if (!embeddingFile.isEmpty()) { + QFile f(embeddingFile); + if (!f.open(QIODevice::ReadOnly|QIODevice::Text)) { out << "cannot read embedding file\n"; return false; } + const auto doc = QJsonDocument::fromJson(f.readAll()); + if (!doc.isArray()) { out << "embedding file must be JSON array\n"; return false; } + for (const auto &v : doc.array()) vec.push_back(static_cast(v.toDouble())); + } + // Hybrid: try text first, then vector. + auto rows = dal.searchText(ns->id, text.toStdString(), k); + int printed = 0; + for (size_t i=0; iid, vec, k-printed); + for (const auto &p : more) { + auto item = dal.getItemById(p.first); + if (!item) continue; + out << QString::fromStdString(p.first) << '\t' << QString::fromStdString(item->text.value_or("")) << '\t' << QString::number(p.second, 'f', 3) << "\n"; + ++printed; if (printed>=k) break; + } + } + return true; +} + } // namespace int main(int argc, char** argv) { @@ -711,6 +830,43 @@ int main(int argc, char** argv) { "dsn"); parser.addOption(dsnOption); + // MCP server mode + QCommandLineOption mcpServeOption(QStringList() << "S" << "mcp-serve", + "Run as an MCP server instead of a one-shot tool. Optional backend name (stdio|ws).", + "backend", "stdio"); + parser.addOption(mcpServeOption); + QCommandLineOption mcpAddrOption(QStringList() << "A" << "mcp-address", + "Address to listen on for network backends.", + "address", "127.0.0.1:8000"); + parser.addOption(mcpAddrOption); + + // DB navigation helpers + QCommandLineOption dbNsOption(QStringList() << "--db-namespaces", + "List namespaces in the database and exit."); + parser.addOption(dbNsOption); + QCommandLineOption dbItemsOption(QStringList() << "--db-items", + "List recent items in a namespace (requires --ns). Optional --limit."); + parser.addOption(dbItemsOption); + QCommandLineOption nsNameOption(QStringList() << "--ns", + "Namespace name for DB operations.", + "name"); + parser.addOption(nsNameOption); + QCommandLineOption limitOption(QStringList() << "--limit", + "Limit for DB operations (default 10).", + "n", "10"); + parser.addOption(limitOption); + QCommandLineOption dbSearchOption(QStringList() << "--db-search", + "Hybrid search in a namespace. Use --text and/or --embedding-file. Requires --ns."); + parser.addOption(dbSearchOption); + QCommandLineOption textOption(QStringList() << "--text", + "Text query for DB search.", + "q"); + parser.addOption(textOption); + QCommandLineOption embFileOption(QStringList() << "--embedding-file", + "Path to JSON array containing embedding vector for hybrid search.", + "path"); + parser.addOption(embFileOption); + parser.addPositionalArgument("tool", "Tool name to invoke."); parser.addPositionalArgument("payload", "Optional JSON payload or file path (use '-' for stdin).", "[payload]"); @@ -723,6 +879,7 @@ int main(int argc, char** argv) { const bool verbose = parser.isSet(verboseOption); const bool interactive = parser.isSet(interactiveOption); const bool initRequested = parser.isSet(initOption); + const bool runMcp = parser.isSet(mcpServeOption); std::optional configDsn = readDsnFromConfig(); const char* envDsn = std::getenv("PG_DSN"); @@ -768,6 +925,32 @@ int main(int argc, char** argv) { return 0; } + // MCP server mode first (exclusive) + if (runMcp) { + const QString backend = parser.value(mcpServeOption); + const QString addr = parser.value(mcpAddrOption); + return runMcpServer(backend, addr, qerr); + } + + // DB inspection helpers (exclusive) + if (parser.isSet(dbNsOption)) { + return dbListNamespaces(qout) ? 0 : 1; + } + if (parser.isSet(dbItemsOption)) { + const QString ns = parser.value(nsNameOption); + if (ns.isEmpty()) { qerr << "--db-items requires --ns \n"; return 1; } + bool ok=false; int lim = parser.value(limitOption).toInt(&ok); if (!ok || lim<=0) lim=10; + return dbListItems(ns, lim, qout) ? 0 : 1; + } + if (parser.isSet(dbSearchOption)) { + const QString ns = parser.value(nsNameOption); + if (ns.isEmpty()) { qerr << "--db-search requires --ns \n"; return 1; } + bool ok=false; int k = parser.value(limitOption).toInt(&ok); if (!ok || k<=0) k=5; + const QString text = parser.value(textOption); + const QString embPath = parser.value(embFileOption); + return dbSearch(ns, text, embPath, k, qout) ? 0 : 1; + } + const QStringList positional = parser.positionalArguments(); if (positional.isEmpty()) { parser.showHelp(1); @@ -808,4 +991,4 @@ int main(int argc, char** argv) { } std::cout << response << std::endl; return 0; -} \ No newline at end of file +} diff --git a/src/cli/py_embedder.py b/src/cli/py_embedder.py new file mode 100644 index 0000000..627fe35 --- /dev/null +++ b/src/cli/py_embedder.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +""" +Lightweight embedding helper moved from ingest/ for transparency. + +Usage examples: + - Single embedding via Ollama: + OLLAMA_BASE=http://localhost:11434 \ + ./py_embedder.py --model bge-m3:latest --text "hello world" + + - Batch from stdin (one line per text): + ./py_embedder.py --model bge-m3:latest --stdin < texts.txt + +Outputs JSON array of floats (for single text) or array-of-arrays for batches. +This script does not touch the database; it only produces vectors. +""" +import os, sys, json, argparse, requests + +def embed_ollama(texts, model, base): + url = f"{base}/api/embeddings" + # Some Ollama models accept a single prompt; do one-by-one for reliability + out = [] + for t in texts: + r = requests.post(url, json={"model": model, "prompt": t}, timeout=120) + r.raise_for_status() + data = r.json() + if "embedding" in data: + out.append(data["embedding"]) # single vector + elif "embeddings" in data: + out.extend(data["embeddings"]) # multiple vectors + else: + raise RuntimeError("Embedding response missing 'embedding(s)'") + return out + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--model", default=os.environ.get("EMBED_MODEL","bge-m3:latest")) + ap.add_argument("--text", help="Text to embed; if omitted, use --stdin") + ap.add_argument("--stdin", action="store_true", help="Read texts from stdin (one per line)") + ap.add_argument("--base", default=os.environ.get("OLLAMA_BASE","http://localhost:11434")) + args = ap.parse_args() + + texts = [] + if args.text: + texts = [args.text] + elif args.stdin: + texts = [line.rstrip("\n") for line in sys.stdin if line.strip()] + else: + ap.error("Provide --text or --stdin") + + vectors = embed_ollama(texts, args.model, args.base) + if len(texts) == 1 and vectors: + print(json.dumps(vectors[0])) + else: + print(json.dumps(vectors)) + +if __name__ == "__main__": + main() + diff --git a/src/middleware/orchestrator.cpp b/src/middleware/orchestrator.cpp new file mode 100644 index 0000000..a90cab2 --- /dev/null +++ b/src/middleware/orchestrator.cpp @@ -0,0 +1,172 @@ +#include "orchestrator.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// ---------- OllamaModelProvider ---------- +OllamaModelProvider::OllamaModelProvider(QObject *parent) + : QObject(parent) +{ + const auto env = QProcessEnvironment::systemEnvironment(); + baseUrl_ = env.value(QStringLiteral("OLLAMA_BASE"), QStringLiteral("http://localhost:11434")); +} + +QString OllamaModelProvider::chooseModelForAspect(const QString &aspect) const { + // Simple mapping; could read models.yaml in the future. + if (aspect.compare(QStringLiteral("companion"), Qt::CaseInsensitive) == 0) return defaultModel_; + if (aspect.compare(QStringLiteral("code"), Qt::CaseInsensitive) == 0) return defaultModel_; + return defaultModel_; +} + +QString OllamaModelProvider::generate(const QString &prompt, const QString &aspect) { + const QString model = chooseModelForAspect(aspect); + const QUrl url(baseUrl_ + QStringLiteral("/api/generate")); + + QNetworkRequest req(url); + req.setHeader(QNetworkRequest::ContentTypeHeader, QStringLiteral("application/json")); + const QJsonObject body{ + {QStringLiteral("model"), model}, + {QStringLiteral("prompt"), prompt}, + {QStringLiteral("stream"), false}, + }; + const QByteArray payload = QJsonDocument(body).toJson(QJsonDocument::Compact); + QEventLoop loop; + QNetworkReply *reply = nam_.post(req, payload); + QObject::connect(reply, &QNetworkReply::finished, &loop, &QEventLoop::quit); + // Time out defensively after 10s; return empty on failure. + QTimer to; + to.setSingleShot(true); + QObject::connect(&to, &QTimer::timeout, &loop, &QEventLoop::quit); + to.start(10000); + loop.exec(); + if (reply->error() != QNetworkReply::NoError) { + reply->deleteLater(); + return QString(); + } + const auto data = reply->readAll(); + reply->deleteLater(); + const auto doc = QJsonDocument::fromJson(data); + if (!doc.isObject()) return QString(); + return doc.object().value(QStringLiteral("response")).toString().trimmed(); +} + +// ---------- Orchestrator ---------- +Orchestrator::Orchestrator(QObject *parent) + : QObject(parent) +{ + // Default provider (can be replaced in tests) + static OllamaModelProvider defaultProv; // lifetime: process + model_ = &defaultProv; + + connect(&timer_, &QTimer::timeout, this, &Orchestrator::processPendingTasks); +} + +void Orchestrator::start(int intervalMs) { + ensureResolvedDirs(); + continuityHandshakeOnce(); + timer_.start(intervalMs); +} + +void Orchestrator::stop() { timer_.stop(); } + +void Orchestrator::ensureResolvedDirs() { + if (!stateDir_.exists()) { + const auto env = QProcessEnvironment::systemEnvironment(); + const auto xdgState = env.value(QStringLiteral("XDG_STATE_HOME"), QDir::home().filePath(".local/state")); + stateDir_.setPath(QDir(xdgState).filePath("kompanion")); + } + if (!configDir_.exists()) { + const auto env = QProcessEnvironment::systemEnvironment(); + const auto xdgConf = env.value(QStringLiteral("XDG_CONFIG_HOME"), QDir::home().filePath(".config")); + configDir_.setPath(QDir(xdgConf).filePath("kompanion")); + } + QDir().mkpath(stateDir_.absolutePath()); + QDir().mkpath(journalDirPath()); +} + +QString Orchestrator::nowUtc() const { + return QDateTime::currentDateTimeUtc().toString(Qt::ISODateWithMs).replace(QLatin1Char('+'), QLatin1Char('Z')); +} + +void Orchestrator::ledgerAppend(const QJsonObject &evt) { + QFile f(ledgerPath()); + if (f.open(QIODevice::ReadOnly)) { + // noop: we could hash prev line like the python version; keep minimal for now + f.close(); + } + if (f.open(QIODevice::Append | QIODevice::Text)) { + QJsonObject copy = evt; + copy.insert(QStringLiteral("ts"), nowUtc()); + const QByteArray line = QJsonDocument(copy).toJson(QJsonDocument::Compact) + '\n'; + f.write(line); + f.close(); + } +} + +void Orchestrator::journalAppend(const QString &text) { + // Ensure journal directory exists even when start() was not called (tests) + QDir().mkpath(journalDirPath()); + const QString file = QDir(journalDirPath()).filePath(QDate::currentDate().toString(Qt::ISODate) + QStringLiteral(".md")); + QFile f(file); + if (f.open(QIODevice::Append | QIODevice::Text)) { + QTextStream out(&f); + out << "- " << nowUtc() << ' ' << text << '\n'; + out.flush(); + f.close(); + } + QJsonObject evt{{QStringLiteral("actor"), QStringLiteral("Χγφτ")}, {QStringLiteral("action"), QStringLiteral("journal.append")}}; + ledgerAppend(evt); +} + +void Orchestrator::continuityHandshakeOnce() { + if (continuityDone_) return; + continuityDone_ = true; + QJsonObject evt{{QStringLiteral("actor"), QStringLiteral("Χγφτ")}, {QStringLiteral("action"), QStringLiteral("CONTINUITY_ACCEPTED")}}; + ledgerAppend(evt); +} + +void Orchestrator::handleJournalFromPrompt(const QJsonObject &obj) { + const QString aspect = obj.value(QStringLiteral("aspect")).toString(QStringLiteral("companion")); + const QString prompt = obj.value(QStringLiteral("prompt")).toString(); + if (!model_) return; + const QString preface = QStringLiteral("Write a brief, warm reflection.\nPrompt:\n"); + const QString out = model_->generate(preface + prompt, aspect); + if (!out.isEmpty()) { + journalAppend(out); + } + emit taskProcessed(QStringLiteral("journal.from_prompt")); +} + +void Orchestrator::processPendingTasks() { + QFile f(tasksPath()); + if (!f.exists()) return; + if (!f.open(QIODevice::ReadOnly | QIODevice::Text)) return; + const QByteArray data = f.readAll(); + f.close(); + // Truncate after reading + if (f.open(QIODevice::WriteOnly | QIODevice::Truncate)) f.close(); + + const QList lines = QByteArray(data).split('\n'); + for (const QByteArray &raw : lines) { + const QByteArray trimmed = raw.trimmed(); + if (trimmed.isEmpty()) continue; + const auto doc = QJsonDocument::fromJson(trimmed); + if (!doc.isObject()) continue; + const QJsonObject obj = doc.object(); + const QString type = obj.value(QStringLiteral("type")).toString(); + if (type == QStringLiteral("journal.from_prompt")) { + handleJournalFromPrompt(obj); + } else { + QJsonObject evt{{QStringLiteral("action"), QStringLiteral("unknown.task")}, {QStringLiteral("type"), type}}; + ledgerAppend(evt); + } + } +} diff --git a/src/middleware/orchestrator.h b/src/middleware/orchestrator.h new file mode 100644 index 0000000..43658fe --- /dev/null +++ b/src/middleware/orchestrator.h @@ -0,0 +1,91 @@ +#pragma once +#include +#include +#include +#include +#include +#include + +#include "kompanion_mw_export.h" + +// Minimal model provider interface so tests can stub generation. +class IModelProvider { +public: + virtual ~IModelProvider() = default; + virtual QString generate(const QString &prompt, const QString &aspect) = 0; +}; + +// Default Ollama-backed provider. Uses OLLAMA_BASE and simple /api/generate call. +class OllamaModelProvider : public QObject, public IModelProvider { + Q_OBJECT +public: + explicit OllamaModelProvider(QObject *parent=nullptr); + QString generate(const QString &prompt, const QString &aspect) override; + void setBaseUrl(const QString &base) { baseUrl_ = base; } + void setDefaultModel(const QString &m) { defaultModel_ = m; } +private: + QString chooseModelForAspect(const QString &aspect) const; // simple heuristic + QString baseUrl_; + QString defaultModel_ = QStringLiteral("qwen2.5:7b"); + QNetworkAccessManager nam_; +}; + +// Simple stub provider used by tests; returns deterministic text. +class StubModelProvider : public IModelProvider { +public: + explicit StubModelProvider(QString canned) : canned_(std::move(canned)) {} + QString generate(const QString &prompt, const QString &aspect) override { + Q_UNUSED(prompt); Q_UNUSED(aspect); return canned_; + } +private: + QString canned_; +}; + +// Orchestrator: replicates runtime/kom_runner.py behaviors in C++. +// - Watches a JSONL tasks file under XDG_STATE_HOME/kompanion +// - Processes tasks like {"type":"journal.from_prompt", "prompt":"...", "aspect":"companion"} +// - Appends to journal (/journal/YYYY-MM-DD.md) and to a simple ledger JSONL +class KOMPANION_MW_EXPORT Orchestrator : public QObject { + Q_OBJECT +public: + explicit Orchestrator(QObject *parent=nullptr); + + // Injectable model provider (Ollama by default). Ownership left to caller. + void setModelProvider(IModelProvider *prov) { model_ = prov; } + + // Directories resolved from XDG_* on start(); overridable for tests. + void setStateDir(const QDir &dir) { stateDir_ = dir; } + void setConfigDir(const QDir &dir) { configDir_ = dir; } + + // Poll loop control. + void start(int intervalMs = 3000); + void stop(); + + // One-shot tick (public for tests). + void processPendingTasks(); + +signals: + void taskProcessed(const QString &kind); + +private: + void ensureResolvedDirs(); + void continuityHandshakeOnce(); + void ledgerAppend(const QJsonObject &evt); + void journalAppend(const QString &line); // Also emits ledger entry + QString nowUtc() const; + + // Task handlers + void handleJournalFromPrompt(const QJsonObject &obj); + + // Helpers + QString tasksPath() const { return stateDir_.filePath("tasks.jsonl"); } + QString journalDirPath() const { return stateDir_.filePath("journal"); } + QString ledgerPath() const { return stateDir_.filePath("trust_ledger.jsonl"); } + + QDir stateDir_; + QDir configDir_; + QTimer timer_; + bool continuityDone_ = false; + IModelProvider *model_ = nullptr; // not owned +}; + diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d66861e..cd17fe0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -5,3 +5,9 @@ qt_add_executable(test_mw find_package(Qt6 REQUIRED COMPONENTS Core Test) target_link_libraries(test_mw PRIVATE Qt6::Core Qt6::Test kompanion_mw) add_test(NAME test_mw COMMAND test_mw) + +qt_add_executable(test_orchestrator + test_orchestrator.cpp +) +target_link_libraries(test_orchestrator PRIVATE Qt6::Core Qt6::Network Qt6::Test kompanion_mw) +add_test(NAME test_orchestrator COMMAND test_orchestrator) diff --git a/tests/test_orchestrator.cpp b/tests/test_orchestrator.cpp new file mode 100644 index 0000000..e043b57 --- /dev/null +++ b/tests/test_orchestrator.cpp @@ -0,0 +1,51 @@ +#include +#include +#include +#include + +#include "../src/middleware/orchestrator.h" + +class OrchestratorTest : public QObject { + Q_OBJECT +private slots: + void journal_from_prompt_writes_outputs(); +}; + +void OrchestratorTest::journal_from_prompt_writes_outputs() { + // Create a temp state dir + QDir tmp = QDir::temp(); + const QString base = QStringLiteral("kompanion_test_%1").arg(QDateTime::currentMSecsSinceEpoch()); + QVERIFY(tmp.mkpath(base)); + QDir state(tmp.filePath(base)); + + // Prepare a task JSONL + QFile tasks(state.filePath("tasks.jsonl")); + QVERIFY(tasks.open(QIODevice::WriteOnly | QIODevice::Truncate | QIODevice::Text)); + const QByteArray line = QByteArray("{\"type\":\"journal.from_prompt\",\"aspect\":\"companion\",\"prompt\":\"hello world\"}\n"); + QVERIFY(tasks.write(line) == line.size()); + tasks.close(); + + // Stub provider returns deterministic text + StubModelProvider stub(QStringLiteral("TEST_OUTPUT")); + + Orchestrator orch; + orch.setStateDir(state); + orch.setModelProvider(&stub); + orch.processPendingTasks(); + + // Expect journal file for today exists and contains the output + const QString journalPath = state.filePath("journal/" + QDate::currentDate().toString(Qt::ISODate) + ".md"); + QFile journal(journalPath); + QVERIFY(journal.exists()); + QVERIFY(journal.open(QIODevice::ReadOnly | QIODevice::Text)); + const QString content = QString::fromUtf8(journal.readAll()); + QVERIFY2(content.contains("TEST_OUTPUT"), "Journal should contain model output"); + + // Expect ledger file exists + QFile ledger(state.filePath("trust_ledger.jsonl")); + QVERIFY(ledger.exists()); +} + +QTEST_MAIN(OrchestratorTest) +#include "test_orchestrator.moc" + diff --git a/tools/pg_search.py b/tools/pg_search.py deleted file mode 100644 index 28faf9c..0000000 --- a/tools/pg_search.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python3 -import os, sys, json, requests, psycopg - -DB=os.environ.get("DB_URL","dbname=kompanion user=kompanion host=/var/run/postgresql") -OLLAMA=os.environ.get("OLLAMA_BASE","http://127.0.0.1:11434") -MODEL=os.environ.get("EMBED_MODEL","mxbai-embed-large") -SPACE=os.environ.get("EMBED_SPACE","dev_knowledge") - -HELP="""\ -Usage: pg_search.py "query text" [k] -Env: DB_URL, OLLAMA_BASE, EMBED_MODEL, EMBED_SPACE (default dev_knowledge) -Prints JSON results: [{score, uri, lineno, text}]. -""" - -def embed(q: str): - r = requests.post(f"{OLLAMA}/api/embeddings", json={"model": MODEL, "prompt": q}, timeout=120) - r.raise_for_status() - return r.json()["embedding"] - -if __name__=="__main__": - if len(sys.argv)<2: - print(HELP, file=sys.stderr); sys.exit(1) - query = sys.argv[1] - k = int(sys.argv[2]) if len(sys.argv)>2 else 8 - vec = embed(query) - with psycopg.connect(DB) as conn, conn.cursor() as cur: - cur.execute("SELECT id, dim FROM komp.space WHERE name=%s", (SPACE,)) - row = cur.fetchone() - if not row: - sys.exit(f"space {SPACE} missing") - sid, dim = row - if dim not in (768,1024): - sys.exit(f"unsupported dim {dim}") - table = f"komp.embedding_{dim}" - # cosine distance with vector_cosine_ops - sql = f""" - SELECT (e.embedding <=> %(v)s::vector) AS score, s.uri, k.lineno, k.text - FROM {table} e - JOIN komp.chunk k ON k.id = e.chunk_id - JOIN komp.source s ON s.id = k.source_id - WHERE e.space_id = %(sid)s - ORDER BY e.embedding <=> %(v)s::vector - LIMIT %(k)s - """ - cur.execute(sql, {"v": vec, "sid": sid, "k": k}) - out=[{"score":float(r[0]),"uri":r[1],"lineno":r[2],"text":r[3]} for r in cur.fetchall()] - print(json.dumps(out, ensure_ascii=False, indent=2)) -