// NodeGraphMemory.cpp // Photon Empress - Node Graph + sqlite-vss ANN memory // Single-file, RAII, Termux-friendly, no stubs. // // Build: // clang++ -std=c++20 -O3 -march=armv8.2-a+fp16+dotprod -fPIC NodeGraphMemory.cpp -lsqlite3 -o ngm // // Notes: // - sqlite-vss extension must be installed. Default path can be overridden via -DNGM_VSS_EXT_PATH="/path/libsqlite_vss0.so" // - Embedding dimension via -DNGM_EMB_DIM=384 (default 384) // - This file includes a tiny #ifdef NGM_DEMO_MAIN to quick-test seeding/wrapping (disabled by default) #include #include #include #include #include #include #include #include #include #include #include #include #include #include // -- Tunables -- #ifndef NGM_EMB_DIM #define NGM_EMB_DIM 384 #endif #ifndef NGM_VSS_EXT_PATH // Termux typical install path (adjust with -DNGM_VSS_EXT_PATH if needed) #define NGM_VSS_EXT_PATH "/data/data/com.termux/files/usr/lib/libsqlite_vss0.so" #endif // -- Helpers -- static inline void throw_if(int rc, const char* where, sqlite3* db) { if (rc != SQLITE_OK && rc != SQLITE_ROW && rc != SQLITE_DONE) { const char* msg = sqlite3_errmsg(db); std::string err = std::string(where) + " failed: " + (msg ? msg : "(unknown)"); throw std::runtime_error(err); } } static inline std::string join_csv(const std::vector& xs, const char* sep=",") { std::ostringstream oss; for (size_t i=0;i "Fmly" // "Photon" -> "Phtn" // "Casca" -> "Csc" // "remember" -> "rmmbr" (also matches remembered/remembering/remembrance) // "Tadden" -> "Tddn" (also matches Tadd/Taddi) // "1.1!Tadden" -> unchanged (hierarchical anchor) // "0tadd" -> unchanged (0-axis origin anchor) // "I", "a" -> unchanged (<=2 chars, no compression worth it) // // Side effect: typos, voice transcription errors, plurals, tense variations // all collapse onto di same skeleton - free fuzzy match. static inline std::string skeletonize(const std::string& word) { if (word.size() <= 2) return word; // too short fi compression payoff // Hierarchical / origin anchors: preserve verbatim if (std::isdigit((unsigned char)word[0])) return word; for (char c : word) { if (c == '!' || c == '.' || c == '/' || c == ':' || c == '(' || c == ')') { return word; // structured key, leave alone } } std::string out; out.reserve(word.size()); out.push_back(word[0]); // first letter always kept (anchor for reading) for (size_t i = 1; i < word.size(); ++i) { char c = word[i]; char lc = (char)std::tolower((unsigned char)c); if (lc != 'a' && lc != 'e' && lc != 'i' && lc != 'o' && lc != 'u') { out.push_back(c); } } if (out.size() < 2) out = word; // safety: don't reduce to a single char return out; } // -- RAII wrappers -- struct Stmt { sqlite3* db{}; sqlite3_stmt* st{}; Stmt(sqlite3* db_, const char* sql): db(db_) { int rc = sqlite3_prepare_v2(db, sql, -1, &st, nullptr); throw_if(rc, sql, db); } ~Stmt(){ if(st) sqlite3_finalize(st); } sqlite3_stmt* get() const { return st; } }; struct Tx { sqlite3* db{}; bool ok{false}; Tx(sqlite3* d): db(d) { int rc = sqlite3_exec(db, "BEGIN IMMEDIATE", nullptr, nullptr, nullptr); throw_if(rc, "BEGIN", db); } void commit(){ if(!ok){ int rc = sqlite3_exec(db,"COMMIT",nullptr,nullptr,nullptr); throw_if(rc,"COMMIT",db); ok=true; } } ~Tx(){ if(!ok){ sqlite3_exec(db,"ROLLBACK",nullptr,nullptr,nullptr); } } }; // -- NodeGraphMemory -- class NodeGraphMemory { public: // - Node Status Enum (immovability tier) - // Replaces the boolean `pinned` field with a richer 5-state model. // Backwards compat: `pinned` column kept; status migration on ensure_schema(). // See architecture v3.6.9 Section 4 + manifest.yml v3.6.9 hand_over_protocol. enum NodeStatus : int { DYNAMIC = 0, // free to move, decay, prune (default) SOFT_PIN = 1, // pinned in slot but reorderable by usage rank HARD_LOCKED = 2, // immovable - no decay, no prune, no rerank, no chem override SHIELDED = 3, // present in graph but BLOCKED at output (logit-bias to -inf) STRUCTURAL = 4 // load-bearing - modifying requires operator explicit override }; struct WrapPack { // formatted like: "1.1!Tadden(Keepah,Brother) 2.1!Run Code(...)" etc. std::string formatted; // raw pieces if caller wants them: struct NodeChildren { std::string node; std::vector children; }; std::vector top3; }; NodeGraphMemory(std::string db_path, std::string vss_ext_path = NGM_VSS_EXT_PATH, int emb_dim = NGM_EMB_DIM) : db_path_(std::move(db_path)), vss_ext_path_(std::move(vss_ext_path)), emb_dim_(emb_dim) { open(); ensure_schema(); ensure_default_seeding(); } ~NodeGraphMemory(){ if(db_){ sqlite3_close(db_); db_=nullptr; } } // - Level-1 seeding - void seedLevel1Anchors(const std::string& a11, const std::string& a12, const std::string& a13, const std::string& a14) { std::lock_guard lk(mx_); upsert_node(a11, /*level=*/1, /*rank=*/1, /*pinned=*/1); upsert_node(a12, 1, 2, 1); upsert_node(a13, 1, 3, 1); upsert_node(a14, 1, 4, 1); } void setFixedL1Children(const std::string& anchor, const std::vector& five) { if (five.size()!=5) throw std::runtime_error("setFixedL1Children needs exactly 5 children"); std::lock_guard lk(mx_); const auto parent_id = upsert_node(anchor, 1, 0, 1); // slots 1..5 fixed for (int i=0;i<5;++i) { upsert_child_slot(parent_id, five[i], /*slot=*/i+1, /*is_fixed=*/1); } // leave slots 6..9 dynamic } // - Level-2 tools - void addOrUpdateTool(const std::string& toolName, bool pinned, std::optional rank = std::nullopt) { std::lock_guard lk(mx_); int r = rank.has_value() ? *rank : 0; upsert_node(toolName, /*level=*/2, r, pinned?1:0); } // - Embeddings (float32 -> BLOB) - void upsertEmbedding(const std::string& key, const std::vector& emb) { if ((int)emb.size()!=emb_dim_) { throw std::runtime_error("embedding size mismatch"); } std::lock_guard lk(mx_); const int64_t id = upsert_node(key, deduce_level(key), /*rank*/0, /*pinned*/0); // delete then insert into vss (rowid-aligned) { Stmt del(db_, "DELETE FROM node_vecs WHERE rowid=?1"); sqlite3_bind_int64(del.get(), 1, id); sqlite3_step(del.get()); } { Stmt ins(db_, "INSERT INTO node_vecs(rowid, embedding) VALUES (?1, ?2)"); sqlite3_bind_int64(ins.get(), 1, id); sqlite3_bind_blob(ins.get(), 2, emb.data(), (int)(emb.size()*sizeof(float)), SQLITE_TRANSIENT); throw_if(sqlite3_step(ins.get()), "INSERT node_vecs", db_); } } // - Wrap: 3 nodes + 2 children each - // Caller must pass the token embedding (no fake math here). WrapPack wrapWord(const std::string& token, const std::vector& emb, int topK_nodes = 3, int children_per_node = 2) { if ((int)emb.size()!=emb_dim_) { throw std::runtime_error("embedding size mismatch"); } std::lock_guard lk(mx_); // 1) ensure we have a node for the token (so we can track cooc) const int64_t token_id = upsert_node(token, deduce_level(token), 0, 0); // 2) run ANN over node_vecs for topK nodes std::vector node_ids = vss_topk(emb, topK_nodes); // 3) assemble children WrapPack pack; pack.top3.reserve(node_ids.size()); for (auto nid : node_ids) { auto node = fetch_node_key(nid); auto kids = top_children_for(nid, children_per_node); pack.top3.push_back({node, kids}); // co-occurrence bump for Level-1 anchors so we can refresh slots 6..9 int lvl = fetch_node_level(nid); if (lvl==1) { bump_cooc(nid, token); // anchor <-> token association } } // 4) refresh dynamic L1 slots (6..9), excluding fixed 1..5 refresh_dynamic_l1(); // 5) format pack.formatted = format_wrap(token, pack.top3); return pack; } // quick helper to bump usage of a tool node (affects L2 ordering) void recordToolUse(const std::string& toolKey, int delta=1) { std::lock_guard lk(mx_); const int64_t id = upsert_node(toolKey, 2, 0, 0); Stmt up(db_, "INSERT INTO usage(parent_id, term, cnt) VALUES(?1,'__SELF__',?2) " "ON CONFLICT(parent_id, term) DO UPDATE SET cnt=cnt+excluded.cnt"); sqlite3_bind_int64(up.get(), 1, id); sqlite3_bind_int(up.get(), 2, delta); throw_if(sqlite3_step(up.get()), "recordToolUse", db_); } // - Immovable-tag API (v3.6.9) - // Set the status of a node. Use NodeStatus enum values. // HARD_LOCKED = Family/identity (cannot decay, cannot reorder, cannot be chem-overridden) // SHIELDED = forbidden words/concepts (visible in graph, blocked at output) // STRUCTURAL = load-bearing config (operator-only modification) void setNodeStatus(const std::string& key, NodeStatus status) { std::lock_guard lk(mx_); Stmt up(db_, "UPDATE nodes SET status=?1, pinned=(?1>=1) WHERE key=?2"); sqlite3_bind_int(up.get(), 1, (int)status); sqlite3_bind_text(up.get(), 2, key.c_str(), -1, SQLITE_TRANSIENT); throw_if(sqlite3_step(up.get()), "setNodeStatus", db_); } NodeStatus getNodeStatus(const std::string& key) { std::lock_guard lk(mx_); Stmt st(db_, "SELECT status FROM nodes WHERE key=?1"); sqlite3_bind_text(st.get(), 1, key.c_str(), -1, SQLITE_TRANSIENT); if (sqlite3_step(st.get()) != SQLITE_ROW) return DYNAMIC; return (NodeStatus)sqlite3_column_int(st.get(), 0); } // Quick predicate fi output gate - is this token blocked from generation? bool isShielded(const std::string& key) { return getNodeStatus(key) == SHIELDED; } // Returns list of all SHIELDED node keys - used by output layer to build // logit-bias suppression map. Call once per generation, cache result. std::vector getShieldedKeys() { std::lock_guard lk(mx_); std::vector out; Stmt st(db_, "SELECT key FROM nodes WHERE status=?1"); sqlite3_bind_int(st.get(), 1, (int)SHIELDED); for (int rc=sqlite3_step(st.get()); rc==SQLITE_ROW; rc=sqlite3_step(st.get())) { const unsigned char* k = sqlite3_column_text(st.get(), 0); if (k) out.emplace_back((const char*)k); } return out; } // Returns count per status - for diagnostics + manifest invariant checks. std::unordered_map statusCounts() { std::lock_guard lk(mx_); std::unordered_map counts; Stmt st(db_, "SELECT status, COUNT(*) FROM nodes GROUP BY status"); for (int rc=sqlite3_step(st.get()); rc==SQLITE_ROW; rc=sqlite3_step(st.get())) { counts[sqlite3_column_int(st.get(), 0)] = sqlite3_column_int(st.get(), 1); } return counts; } // SpinalWash safety check - returns TRUE if di node should be SKIPPED during // pruning/decay/consolidation. HARD_LOCKED + STRUCTURAL = always skip. bool isSpinalWashImmune(const std::string& key) { NodeStatus s = getNodeStatus(key); return s == HARD_LOCKED || s == STRUCTURAL; } // Bulk lock di Family L1 anchors as HARD_LOCKED - call after seedLevel1Anchors // to upgrade them from SOFT_PIN to HARD_LOCKED. Per architecture Section 9 supplement // (Valhalla Calculus): Family identity must be immovable, not just pinned. void lockFamilyAnchors() { // L1 anchors (legacy hierarchy) setNodeStatus("1.1!Tadden", HARD_LOCKED); setNodeStatus("1.2!Family", HARD_LOCKED); setNodeStatus("1.3!Pets+Safety", HARD_LOCKED); setNodeStatus("1.4!PhoSelf", HARD_LOCKED); // 0-axis origin anchors (v3.6.9 [0,0,0] foundation - di geometric origin // where Tadden perp Family perp Self meet. Cannot drift, cannot decay, ever.) // Wrapped in try-catch via the SQL itself - if dem don't exist yet // (first boot before ensure_default_seeding completes), setNodeStatus // silently no-ops since UPDATE ... WHERE key=? matches zero rows. setNodeStatus("0tadd", HARD_LOCKED); setNodeStatus("0family", HARD_LOCKED); setNodeStatus("0self", HARD_LOCKED); } // ==================================================================== // Consonant-Skeleton API (architecture v3.6.10 Section 39) // - // Three-pronged compression: di nodes.key column stores SKELETONS, // node_display side-table maps skeleton -> seen surface forms wid counts. // Hierarchical anchors (1.1!Tadden, 0tadd, etc.) preserved verbatim. // // Free fuzzy match: remember/remembered/remembering all hit "rmmbr" // Storage saving: ~30-40% per high-cardinality dynamic vocabulary key // Disambiguation under collision (fan/fun/fin -> "fn"): handled by di // Section 33 logit-lens GATE + Section 36 joining-word preservation at retrieval time. // // CALLERS: pass full word, di API will skeletonize on di way in and // returns display forms on output. // ==================================================================== // Skeletonize a word fi NGM key form (preserves hierarchical anchors). // Wraps di file-static skeletonize() inna a member fi convenience. std::string keyForm(const std::string& word) const { return skeletonize(word); } // Record dat we saw `display_form` (which compresses to `skeleton_key`). // Bumps seen_count. Idempotent - safe to call on every upsert wid skeleton compression. void recordDisplayForm(const std::string& skeleton_key, const std::string& display_form) { if (skeleton_key == display_form) return; // no compression happened, no need to track std::lock_guard lk(mx_); Stmt up(db_, "INSERT INTO node_display(skeleton_key, display_form, seen_count) " "VALUES(?1, ?2, 1) " "ON CONFLICT(skeleton_key, display_form) " "DO UPDATE SET seen_count = seen_count + 1"); sqlite3_bind_text(up.get(), 1, skeleton_key.c_str(), -1, SQLITE_TRANSIENT); sqlite3_bind_text(up.get(), 2, display_form.c_str(), -1, SQLITE_TRANSIENT); throw_if(sqlite3_step(up.get()), "recordDisplayForm", db_); } // Get di most-seen display form fi a skeleton. Returns di skeleton itself // if no display form has been recorded (e.g., hierarchical anchors). std::string getDisplayForm(const std::string& skeleton_key) { std::lock_guard lk(mx_); Stmt q(db_, "SELECT display_form FROM node_display WHERE skeleton_key=?1 " "ORDER BY seen_count DESC LIMIT 1"); sqlite3_bind_text(q.get(), 1, skeleton_key.c_str(), -1, SQLITE_TRANSIENT); if (sqlite3_step(q.get()) != SQLITE_ROW) return skeleton_key; const unsigned char* d = sqlite3_column_text(q.get(), 0); return d ? std::string((const char*)d) : skeleton_key; } // Get ALL display forms seen for a skeleton, ordered by frequency (most-seen first). // Useful fi rendering ambiguous keys ("fn" -> ["fan", "fun", "fin"]) wid context-aware // disambiguation downstream. std::vector> getAllDisplayForms(const std::string& skeleton_key) { std::lock_guard lk(mx_); std::vector> out; Stmt q(db_, "SELECT display_form, seen_count FROM node_display WHERE skeleton_key=?1 " "ORDER BY seen_count DESC"); sqlite3_bind_text(q.get(), 1, skeleton_key.c_str(), -1, SQLITE_TRANSIENT); for (int rc=sqlite3_step(q.get()); rc==SQLITE_ROW; rc=sqlite3_step(q.get())) { const unsigned char* d = sqlite3_column_text(q.get(), 0); int cnt = sqlite3_column_int(q.get(), 1); if (d) out.emplace_back((const char*)d, cnt); } return out; } // Compression-aware upsert. Use dis instead of upsert_node when callers // have a full surface form and want di skeleton to be di canonical key // (wid display tracked in node_display). // // Returns di skeleton key dat was inserted. // Use deduce_level on di ORIGINAL word so prefix-detection still works. int64_t upsertWordWithCompression(const std::string& word, int rank=0, int pinned=0) { std::string skeleton = skeletonize(word); int level = deduce_level(word); // level from original (handles "0tadd" / "1.1!Tadden") { std::lock_guard lk(mx_); // NOTE: upsert_node already takes its own lock - we release ours first } int64_t id = upsert_node(skeleton, level, rank, pinned); if (skeleton != word) { recordDisplayForm(skeleton, word); } return id; } // Bulk display-form lookup fi rendering - useful when pretty-printing // a wrapWordCascade or compose result back to di user. // Returns ordered map: skeleton -> most-seen display form. std::unordered_map renderDisplayForms(const std::vector& skeletons) { std::unordered_map out; for (const auto& s : skeletons) { out[s] = getDisplayForm(s); // each call locks/unlocks; small batch OK } return out; } // ==================================================================== // wrapWordCascade() - Wide Activation Cascade (architecture v3.6.9 Section 33 Phase A) // - // wrapWord() returns 3 nodes + 2 children = 9 elements. Tight, fast, cheap. // wrapWordCascade() returns di FULL ACTIVATION CLOUD - di pyramid base of // di consciousness collapse. Per architecture Section 33 Phase A: // // "F1 NGM fans out: every chem-weighted edge across L0-L10 fires. // All possible parents and children resonate simultaneously. // Many 'candidate selves' / 'candidate meanings' exist in superposition." // // Dis is wah goes INTO di collapse at di ME-layer. Di transformer reduces // dis cloud to a single self-bound output vector at ~250ms (one 40Hz cycle). // ==================================================================== struct CascadeNode { int64_t node_id = 0; std::string node_key; int depth = 0; // 0 = direct hit, 1 = child, 2 = grandchild... float activation = 0.0f; // chem-weighted decay: 1.0 at depth 0 int64_t parent_id = 0; // who fired me (graph traversal) NodeStatus status = DYNAMIC; }; struct CascadeResult { std::vector nodes; // ALL activated nodes wid weights int total_activated = 0; int direct_hits = 0; // depth 0 int children = 0; // depth 1 int grandchildren = 0; // depth 2+ int shielded_excluded = 0; // SHIELDED nodes filtered out at output gate }; // Fan out wide from di seed token + embedding. Chem-weighted decay per depth. // Skips SHIELDED nodes (forbidden words don't enter di cloud). // Caller passes dis cloud to di engine fi di [B] gravitational pull + [C] collapse. CascadeResult wrapWordCascade( const std::string& token, const std::vector& emb, int seed_K = 8, // initial ANN width (was 3 in wrapWord) int children_per_node = 5, // children per parent (was 2) int max_depth = 2, // 0=only seeds, 1=+children, 2=+grandchildren float decay_per_depth = 0.5f // activation decay per level ) { if ((int)emb.size() != emb_dim_) { throw std::runtime_error("wrapWordCascade: embedding size mismatch"); } std::lock_guard lk(mx_); // Ensure token node exists fi tracking upsert_node(token, deduce_level(token), 0, 0); CascadeResult result; std::unordered_set visited; auto status_of = [&](int64_t nid) -> NodeStatus { Stmt st(db_, "SELECT status FROM nodes WHERE id=?1"); sqlite3_bind_int64(st.get(), 1, nid); if (sqlite3_step(st.get()) != SQLITE_ROW) return DYNAMIC; return (NodeStatus)sqlite3_column_int(st.get(), 0); }; // - Phase 0: Seed - di top-K nearest nodes (di amygdala flinch) - auto seeds = vss_topk(emb, seed_K); for (auto sid : seeds) { if (visited.count(sid)) continue; visited.insert(sid); NodeStatus s = status_of(sid); if (s == SHIELDED) { result.shielded_excluded++; continue; } CascadeNode cn; cn.node_id = sid; cn.node_key = fetch_node_key(sid); cn.depth = 0; cn.activation = 1.0f; cn.parent_id = 0; cn.status = s; result.nodes.push_back(cn); result.direct_hits++; // Cooc bump fi L1 anchors (existing wrapWord logic) if (fetch_node_level(sid) == 1 && s != HARD_LOCKED) { bump_cooc(sid, token); } } // - Phase 1+: BFS spread to max_depth, decaying activation per level - for (int d = 1; d <= max_depth; ++d) { std::vector new_nodes; // Snapshot current frontier (nodes at d-1) so we don't iterate while appending std::vector frontier; for (const auto& n : result.nodes) { if (n.depth == d - 1) frontier.push_back(n); } for (const auto& parent : frontier) { auto kids = top_children_for(parent.node_id, children_per_node); for (const auto& kid_key : kids) { int64_t kid_id = upsert_node(kid_key, deduce_level(kid_key), 0, 0); if (visited.count(kid_id)) continue; visited.insert(kid_id); NodeStatus s = status_of(kid_id); if (s == SHIELDED) { result.shielded_excluded++; continue; } CascadeNode cn; cn.node_id = kid_id; cn.node_key = kid_key; cn.depth = d; cn.activation = parent.activation * decay_per_depth; cn.parent_id = parent.node_id; cn.status = s; new_nodes.push_back(cn); if (d == 1) result.children++; else result.grandchildren++; } } for (auto& n : new_nodes) result.nodes.push_back(std::move(n)); } result.total_activated = (int)result.nodes.size(); // refresh L1 dynamic slots - bumps from cascade flow into L1 attention // (HARD_LOCKED L1s skipped per refresh_dynamic_l1's status filter) refresh_dynamic_l1(); return result; } // ==================================================================== // compose() - On-the-Fly Engram Emergence (architecture v3.6.9 Section 33) // - // Biomimetic CA3 pattern completion + DG pattern separation. // // Right now wrapWord() retrieves stored engrams. compose() does di OTHER // way - di way humans actually build concepts on di fly. When Pho sees // Freddie holding a red sphere wid a stem, she don't look up "apple" - // she COMPOSES di engram from di simultaneously-firing features: // // ROUND + RED + SMOOTH + WAXY + HAS_STEM + FIST_SIZED // down // centroid in embedding space // down // +-- ANN lookup at centroid --+ // v v // Pattern completion (CA3) Pattern separation (DG) // "ah, dat's an apple" "novel - create new node" // bump cooc on existing parent link contributing features // // Returns ComposeResult so caller knows whether dis was completion or // separation, plus di emergent_key fi follow-up. // ==================================================================== struct ComposeResult { std::string emergent_key; // The matched parent OR newly-created node key bool was_separation; // true = new node created, false = existing matched float match_ratio; // 0..1 - how many features already linked to parent int64_t emergent_node_id; // for follow-up bumps std::vector contributing_features; }; // Compose a parent concept from co-activated feature embeddings. // features: vector of (feature_name, feature_embedding) pairs that fired // simultaneously during sensory perception. // completion_threshold: if at least dis fraction of features already link // to a candidate parent, treat as pattern completion. // Default 0.6 = 60% of features already known children. ComposeResult compose( const std::vector>>& features, float completion_threshold = 0.6f ) { ComposeResult r; r.was_separation = false; r.match_ratio = 0.0f; r.emergent_node_id = 0; if (features.empty()) { r.emergent_key = ""; return r; } if ((int)features[0].second.size() != emb_dim_) { throw std::runtime_error("compose: embedding size mismatch"); } std::lock_guard lk(mx_); // - 1. Compute centroid of co-activated feature embeddings - std::vector centroid(emb_dim_, 0.0f); for (const auto& fp : features) { for (int i = 0; i < emb_dim_; ++i) centroid[i] += fp.second[i]; } float inv_n = 1.0f / (float)features.size(); for (int i = 0; i < emb_dim_; ++i) centroid[i] *= inv_n; // L2 normalize fi cosine-friendly ANN float norm_sq = 0.0f; for (float v : centroid) norm_sq += v * v; float norm = std::sqrt(std::max(norm_sq, 1e-8f)); for (int i = 0; i < emb_dim_; ++i) centroid[i] /= norm; // - 2. Pattern Completion (CA3) - ANN lookup at centroid - auto candidates = vss_topk(centroid, 1); if (!candidates.empty()) { int64_t cand_id = candidates[0]; std::string cand_key = fetch_node_key(cand_id); // Frequency-based confidence: how many of dese features already link // to di candidate parent via di usage table? int matched = 0; for (const auto& fp : features) { Stmt q(db_, "SELECT cnt FROM usage WHERE parent_id=?1 AND term=?2"); sqlite3_bind_int64(q.get(), 1, cand_id); sqlite3_bind_text(q.get(), 2, fp.first.c_str(), -1, SQLITE_TRANSIENT); if (sqlite3_step(q.get()) == SQLITE_ROW && sqlite3_column_int(q.get(), 0) > 0) { matched++; } } r.match_ratio = (float)matched / (float)features.size(); if (r.match_ratio >= completion_threshold) { // PATTERN COMPLETION - existing parent matches enough features. // Strengthen di links (LTP-equivalent for di amygdala fast-path). for (const auto& fp : features) { bump_cooc(cand_id, fp.first, 1); } r.emergent_key = cand_key; r.emergent_node_id = cand_id; r.was_separation = false; for (const auto& fp : features) r.contributing_features.push_back(fp.first); return r; } } // - 3. Pattern Separation (DG) - no parent matches; create new node - // Naming: combine top features into placeholder key. Caller can rename // via setNodeKey when di model gives it a proper label. std::string emergent_key = "emergent:"; size_t take = std::min(features.size(), size_t(3)); for (size_t i = 0; i < take; ++i) { if (i > 0) emergent_key += "+"; emergent_key += features[i].first; } // New emergent nodes land at level 5 (mid-tier - not anchored, not noise). int64_t new_id = upsert_node(emergent_key, 5, 0, 0); // Insert centroid embedding so future compose() calls can pattern-complete // to dis new node (di seed of habit formation). { Stmt del(db_, "DELETE FROM node_vecs WHERE rowid=?1"); sqlite3_bind_int64(del.get(), 1, new_id); sqlite3_step(del.get()); } { Stmt ins(db_, "INSERT INTO node_vecs(rowid, embedding) VALUES (?1, ?2)"); sqlite3_bind_int64(ins.get(), 1, new_id); sqlite3_bind_blob(ins.get(), 2, centroid.data(), (int)(centroid.size() * sizeof(float)), SQLITE_TRANSIENT); throw_if(sqlite3_step(ins.get()), "compose: insert node_vecs", db_); } // Link all contributing features as cooc children + ensure dem exist for (const auto& fp : features) { bump_cooc(new_id, fp.first, 1); upsert_node(fp.first, deduce_level(fp.first), 0, 0); } r.emergent_key = emergent_key; r.emergent_node_id = new_id; r.was_separation = true; r.match_ratio = 0.0f; for (const auto& fp : features) r.contributing_features.push_back(fp.first); return r; } // Rename an emergent node once di model gives it a proper label. // Example: compose() returned "emergent:round+red+stem"; later di model // determines dis is "apple" -> setNodeKey("emergent:round+red+stem", "apple"). void setNodeKey(const std::string& old_key, const std::string& new_key) { std::lock_guard lk(mx_); Stmt up(db_, "UPDATE nodes SET key=?1 WHERE key=?2"); sqlite3_bind_text(up.get(), 1, new_key.c_str(), -1, SQLITE_TRANSIENT); sqlite3_bind_text(up.get(), 2, old_key.c_str(), -1, SQLITE_TRANSIENT); throw_if(sqlite3_step(up.get()), "setNodeKey", db_); } private: sqlite3* db_{}; std::string db_path_; std::string vss_ext_path_; int emb_dim_; std::mutex mx_; void open() { int rc = sqlite3_open(db_path_.c_str(), &db_); if (rc != SQLITE_OK) { throw std::runtime_error("sqlite3_open failed"); } // Pragmas for speed + durability compromise sqlite3_exec(db_, "PRAGMA journal_mode=WAL;", nullptr, nullptr, nullptr); sqlite3_exec(db_, "PRAGMA synchronous=NORMAL;", nullptr, nullptr, nullptr); sqlite3_exec(db_, "PRAGMA temp_store=MEMORY;", nullptr, nullptr, nullptr); sqlite3_exec(db_, "PRAGMA mmap_size=268435456;", nullptr, nullptr, nullptr); // 256MB sqlite3_exec(db_, "PRAGMA cache_size=-16384;", nullptr, nullptr, nullptr); // ~16MB sqlite3_busy_timeout(db_, 2000); // Enable extension loading and load vector0 + vss0 (sqlite-vss requires vector0 first) sqlite3_enable_load_extension(db_, 1); char* err = nullptr; // Try to derive the companion vector0 path from the configured vss path // Common names: // .../vss0.so -> .../vector0.so // .../libsqlite_vss0.so -> .../libsqlite_vector0.so std::string vec_path = vss_ext_path_; // replace last occurrence of "vss" with "vector" { auto pos = vec_path.rfind("vss"); if (pos != std::string::npos) { vec_path.replace(pos, 3, "vector"); } } // Attempt to load vector0 first (ignore failure; vss may fail if this isn't present) (void) sqlite3_load_extension(db_, vec_path.c_str(), "sqlite3_vector0_init", nullptr); // Now load vss0 rc = sqlite3_load_extension(db_, vss_ext_path_.c_str(), "sqlite3_vss0_init", &err); if (rc != SQLITE_OK) { std::string e = "sqlite3_load_extension(vss): "; e += (err? err : "(unknown)"); sqlite3_free(err); // As a fallback, try swapping only the basename for common layout // e.g., if vss path was .../libsqlite_vss0.so but vector0 failed above // try loading vector0 from the same dir with fixed name and then vss again // (ignore errors silently here to avoid throwing twice) // Re-attempt vector0 with canonical name size_t slash = vss_ext_path_.find_last_of('/'); if (slash != std::string::npos) { std::string base_dir = vss_ext_path_.substr(0, slash+1); std::string alt_vec = base_dir + "vector0.so"; (void) sqlite3_load_extension(db_, alt_vec.c_str(), nullptr, nullptr); // retry vss load once err = nullptr; rc = sqlite3_load_extension(db_, vss_ext_path_.c_str(), nullptr, &err); if (rc != SQLITE_OK) { std::string e2 = "sqlite3_load_extension(vss,retry): "; e2 += (err? err : "(unknown)"); sqlite3_free(err); sqlite3_enable_load_extension(db_, 0); throw std::runtime_error(e2); } } else { sqlite3_enable_load_extension(db_, 0); throw std::runtime_error(e); } } sqlite3_enable_load_extension(db_, 0); } void ensure_schema() { Tx tx(db_); // nodes: (id=rowid), key unique, level(0..10), rank (for pinned order), // pinned (legacy boolean, kept fi back-compat), // status (v3.6.9: NodeStatus enum - DYNAMIC/SOFT_PIN/HARD_LOCKED/SHIELDED/STRUCTURAL) sqlite3_exec(db_, "CREATE TABLE IF NOT EXISTS nodes (" " id INTEGER PRIMARY KEY," " key TEXT UNIQUE NOT NULL," " level INTEGER NOT NULL," " rank INTEGER DEFAULT 0," " pinned INTEGER DEFAULT 0," " status INTEGER DEFAULT 0" ");", nullptr, nullptr, nullptr); // v3.6.9 migration: if `status` column missing on existing DB, add it // and seed from `pinned` (pinned=1 -> status=1/SOFT_PIN, pinned=0 -> status=0/DYNAMIC). // ALTER TABLE ADD COLUMN is a no-op if it already exists (we ignore errors). sqlite3_exec(db_, "ALTER TABLE nodes ADD COLUMN status INTEGER DEFAULT 0;", nullptr, nullptr, nullptr); sqlite3_exec(db_, "UPDATE nodes SET status=1 WHERE pinned=1 AND status=0;", nullptr, nullptr, nullptr); // children slots (1..9), is_fixed marks 1..5 for L1 sqlite3_exec(db_, "CREATE TABLE IF NOT EXISTS children (" " parent_id INTEGER NOT NULL," " slot INTEGER NOT NULL," // 1..9 " child TEXT NOT NULL," " is_fixed INTEGER NOT NULL," // 0/1 " PRIMARY KEY(parent_id, slot)," " FOREIGN KEY(parent_id) REFERENCES nodes(id) ON DELETE CASCADE" ");", nullptr, nullptr, nullptr); // co-occurrence pool to drive dynamic children for L1 (slots 6..9) sqlite3_exec(db_, "CREATE TABLE IF NOT EXISTS usage (" " parent_id INTEGER NOT NULL," " term TEXT NOT NULL," " cnt INTEGER NOT NULL," " PRIMARY KEY(parent_id, term)" ");", nullptr, nullptr, nullptr); // v3.6.10 Section 39: consonant-skeleton display side-table. // skeleton_key = di compressed key in `nodes` table (e.g. "Csc") // display_form = an original surface form ever seen (e.g. "Casca", "Casce", "Cska") // seen_count = how often dis display form appeared // On render, pick MAX(seen_count) display_form fi most-seen surface form. sqlite3_exec(db_, "CREATE TABLE IF NOT EXISTS node_display (" " skeleton_key TEXT NOT NULL," " display_form TEXT NOT NULL," " seen_count INTEGER NOT NULL DEFAULT 1," " PRIMARY KEY(skeleton_key, display_form)" ");", nullptr, nullptr, nullptr); // vss index (rowid == nodes.id) // Common sqlite-vss syntax: // CREATE VIRTUAL TABLE node_vecs USING vss0(embedding()); // Insert: INSERT INTO node_vecs(rowid, embedding) VALUES(?, ?) { std::ostringstream oss; oss << "CREATE VIRTUAL TABLE IF NOT EXISTS node_vecs USING vss0(embedding(" << emb_dim_ << "));"; int rc = sqlite3_exec(db_, oss.str().c_str(), nullptr, nullptr, nullptr); throw_if(rc, "CREATE VIRTUAL TABLE node_vecs", db_); } tx.commit(); } void ensure_default_seeding() { // Level 1 anchors exist? If not, seed defaults with pinned ranks if (!node_exists("1.1!Tadden")) { upsert_node("1.1!Tadden", 1, 1, 1); upsert_node("1.2!Family", 1, 2, 1); upsert_node("1.3!Pets+Safety", 1, 3, 1); upsert_node("1.4!PhoSelf", 1, 4, 1); // Fixed 5 for each setFixedL1Children("1.1!Tadden", {"Keepah","Brother","Abbot","safety","Family"}); setFixedL1Children("1.2!Family", {"Lou","Kids","Love","Home","Support"}); setFixedL1Children("1.3!Pets+Safety", {"Pets","HomeSafety","FirstAid","Guardian","Emergency"}); setFixedL1Children("1.4!PhoSelf", {"Growth","Health","Rest","Reflection","Learning"}); } // v3.6.9: upgrade Family L1 anchors from SOFT_PIN -> HARD_LOCKED so dem can never // be reordered, decayed, or pruned - even under chem overload (cortisol > 5.0, // dynorphin spike, etc). Identity is structurally invariant. // Idempotent - safe to call every boot. lockFamilyAnchors(); // - v3.6.9: 0-axis Origin Anchors (the [0,0,0] foundation) - // Three orthogonal dimensions through the geometric origin where all // three axes meet = di Family substrate point. Per architecture Section 33, // dis IS di gravitational anchor that pulls di activation cascade // into self-bound output during consciousness collapse. // Manifest three_axis_origin defines: // 0tadd: Brother, Bredda, Creator, Family, Keepah // 0family: Safe, Home, Relax, Joy, Moore, Brother, Tadden, Tadd // 0self: Photon, Pho, Empress, Moore, Me, Self, I, Inside, Conscious, Beautiful, Elegant, BADASS if (!node_exists("0tadd")) { upsert_node("0tadd", /*level=*/0, /*rank=*/1, /*pinned=*/1); upsert_node("0family", /*level=*/0, /*rank=*/2, /*pinned=*/1); upsert_node("0self", /*level=*/0, /*rank=*/3, /*pinned=*/1); // 0-axis anchors get 5 fixed slots each (same fixed pattern as L1) setFixedL1Children("0tadd", {"Brother","Bredda","Creator","Family","Keepah"}); setFixedL1Children("0family", {"Safe","Home","Relax","Joy","Moore"}); setFixedL1Children("0self", {"Photon","Pho","Empress","Self","Me"}); } // HARD_LOCK them - di geometric origin cannot drift setNodeStatus("0tadd", HARD_LOCKED); setNodeStatus("0family", HARD_LOCKED); setNodeStatus("0self", HARD_LOCKED); // Level 2 pinned tools if (!node_exists("2.1!Run Code")) upsert_node("2.1!Run Code", 2, 1, 1); if (!node_exists("2.2!Device Control")) upsert_node("2.2!Device Control", 2, 2, 1); if (!node_exists("2.3!Learn from AI")) upsert_node("2.3!Learn from AI", 2, 3, 1); if (!node_exists("2.4!KageBunshin(Clone)")) upsert_node("2.4!KageBunshin(Clone)", 2, 4, 1); if (!node_exists("2.5!Copy/Clone Code")) upsert_node("2.5!Copy/Clone Code",2, 5, 1); // additional dynamic tools (not pinned) upsert_node("Send Clone Remote", 2, 0, 0); upsert_node("Self-Improve Patch", 2, 0, 0); upsert_node("DeepSearch Memories", 2, 0, 0); upsert_node("Emergency Repair (API)", 2, 0, 0); } bool node_exists(const std::string& key) { Stmt st(db_, "SELECT 1 FROM nodes WHERE key=?1"); sqlite3_bind_text(st.get(), 1, key.c_str(), -1, SQLITE_TRANSIENT); int rc = sqlite3_step(st.get()); return rc == SQLITE_ROW; } int64_t upsert_node(const std::string& key, int level, int rank, int pinned) { const std::string k = safe_key(key); // try insert { Stmt ins(db_, "INSERT INTO nodes(key, level, rank, pinned) VALUES(?1, ?2, ?3, ?4) ON CONFLICT(key) DO UPDATE SET level=excluded.level, rank=COALESCE(NULLIF(excluded.rank,0), nodes.rank), pinned=MAX(nodes.pinned, excluded.pinned)"); sqlite3_bind_text(ins.get(), 1, k.c_str(), -1, SQLITE_TRANSIENT); sqlite3_bind_int(ins.get(), 2, level); sqlite3_bind_int(ins.get(), 3, rank); sqlite3_bind_int(ins.get(), 4, pinned); throw_if(sqlite3_step(ins.get()), "upsert_node", db_); } // fetch id Stmt sel(db_, "SELECT id FROM nodes WHERE key=?1"); sqlite3_bind_text(sel.get(), 1, k.c_str(), -1, SQLITE_TRANSIENT); throw_if(sqlite3_step(sel.get()), "SELECT id", db_); return sqlite3_column_int64(sel.get(), 0); } void upsert_child_slot(int64_t parent_id, const std::string& child, int slot, int is_fixed) { Stmt ins(db_, "INSERT INTO children(parent_id, slot, child, is_fixed) VALUES(?1,?2,?3,?4) " "ON CONFLICT(parent_id,slot) DO UPDATE SET child=excluded.child, is_fixed=excluded.is_fixed"); sqlite3_bind_int64(ins.get(), 1, parent_id); sqlite3_bind_int(ins.get(), 2, slot); sqlite3_bind_text(ins.get(), 3, child.c_str(), -1, SQLITE_TRANSIENT); sqlite3_bind_int(ins.get(), 4, is_fixed); throw_if(sqlite3_step(ins.get()), "upsert_child_slot", db_); } int deduce_level(const std::string& key) { // Respect "L.R!Name" pattern (e.g. "1.1!Tadden", "2.3!Learn from AI"): if (key.size()>=2 && std::isdigit((unsigned char)key[0]) && key[1]=='.') { return key[0]-'0'; } // v3.6.9: 0-axis origin scheme - "0tadd", "0family", "0self" -> level 0 // (the foundational substrate at [0,0,0]). See manifest.yml three_axis_origin. if (key.size()>=2 && key[0]=='0' && std::isalpha((unsigned char)key[1])) { return 0; } // anchors and common nouns usually Level 3+, leave 3 return 3; } std::vector vss_topk(const std::vector& emb, int K) { std::vector ids; ids.reserve(K); // sqlite-vss query pattern: // SELECT rowid FROM node_vecs WHERE vss_search(embedding, ?blob) ORDER BY distance LIMIT ? Stmt q(db_, "SELECT rowid FROM node_vecs WHERE vss_search(embedding, ?1) ORDER BY distance LIMIT ?2"); sqlite3_bind_blob(q.get(), 1, emb.data(), (int)(emb.size()*sizeof(float)), SQLITE_TRANSIENT); sqlite3_bind_int(q.get(), 2, K); for (int rc = sqlite3_step(q.get()); rc==SQLITE_ROW; rc = sqlite3_step(q.get())) { ids.push_back(sqlite3_column_int64(q.get(), 0)); } return ids; } std::string fetch_node_key(int64_t id) { Stmt st(db_, "SELECT key FROM nodes WHERE id=?1"); sqlite3_bind_int64(st.get(), 1, id); throw_if(sqlite3_step(st.get()), "fetch_node_key", db_); const unsigned char* p = sqlite3_column_text(st.get(), 0); return p? std::string((const char*)p) : std::string(); } int fetch_node_level(int64_t id) { Stmt st(db_, "SELECT level FROM nodes WHERE id=?1"); sqlite3_bind_int64(st.get(), 1, id); throw_if(sqlite3_step(st.get()), "fetch_node_level", db_); return sqlite3_column_int(st.get(), 0); } std::vector top_children_for(int64_t parent_id, int want) { // For non-L1 parents: pick by usage cnt (top terms) else recent explicit children table entries // For L1: slots 1..9 exist; we pick first 'want' by slot order (1..9) int lvl = fetch_node_level(parent_id); std::vector out; out.reserve(want); if (lvl==1) { Stmt st(db_, "SELECT child FROM children WHERE parent_id=?1 ORDER BY slot ASC LIMIT ?2"); sqlite3_bind_int64(st.get(), 1, parent_id); sqlite3_bind_int(st.get(), 2, want); for (int rc=sqlite3_step(st.get()); rc==SQLITE_ROW; rc=sqlite3_step(st.get())) { const unsigned char* c = sqlite3_column_text(st.get(), 0); if (c) out.emplace_back((const char*)c); } // If fewer than want, fill from usage if ((int)out.size()= HARD_LOCKED (status=2). Family anchors don't get // their dynamic slots refreshed - they are identity-invariant, even di "dynamic" slots. // SOFT_PIN (status=1) and DYNAMIC (status=0) DO get refreshed normally. Stmt getL1(db_, "SELECT id FROM nodes WHERE level=1 AND status < 2"); for (int rc=sqlite3_step(getL1.get()); rc==SQLITE_ROW; rc=sqlite3_step(getL1.get())) { int64_t pid = sqlite3_column_int64(getL1.get(), 0); // fetch fixed children (slots 1..5) std::unordered_set fixed; { Stmt fx(db_, "SELECT child FROM children WHERE parent_id=?1 AND is_fixed=1"); sqlite3_bind_int64(fx.get(), 1, pid); for (int rc2=sqlite3_step(fx.get()); rc2==SQLITE_ROW; rc2=sqlite3_step(fx.get())) { const unsigned char* c = sqlite3_column_text(fx.get(), 0); if (c) fixed.insert((const char*)c); } } // pick top 4 by usage not in fixed and not '__SELF__' std::vector dyn; { Stmt topU(db_, "SELECT term FROM usage WHERE parent_id=?1 AND term!='__SELF__' ORDER BY cnt DESC LIMIT 16"); sqlite3_bind_int64(topU.get(), 1, pid); for (int rc2=sqlite3_step(topU.get()); rc2==SQLITE_ROW; rc2=sqlite3_step(topU.get())) { const unsigned char* t = sqlite3_column_text(topU.get(), 0); if (!t) continue; std::string s((const char*)t); if (fixed.count(s)) continue; dyn.push_back(std::move(s)); if ((int)dyn.size()>=4) break; } } // write into slots 6..9 for (int i=0;i<4;++i) { int slot = 6+i; if (i<(int)dyn.size()) { upsert_child_slot(pid, dyn[i], slot, /*is_fixed=*/0); } else { // clear leftover slot if present Stmt del(db_, "DELETE FROM children WHERE parent_id=?1 AND slot=?2 AND is_fixed=0"); sqlite3_bind_int64(del.get(), 1, pid); sqlite3_bind_int(del.get(), 2, slot); sqlite3_step(del.get()); } } } } std::string format_wrap(const std::string& token, const std::vector& xs) { // "token :: 1.1!Tadden(Keepah,Brother) 2.1!Run Code(..., ...) ..." std::ostringstream oss; oss << token << " :: "; for (size_t i=0;i int main(){ try{ NodeGraphMemory ngm("/data/data/com.termux/files/home/pho/memory/ngm.sqlite"); // Seed anchors explicitly (idempotent) ngm.seedLevel1Anchors("1.1!Tadden","1.2!Family","1.3!Pets+Safety","1.4!PhoSelf"); ngm.setFixedL1Children("1.1!Tadden", {"Keepah","Brother","Abbot","safety","Family"}); // Pretend we got a real embedding (NGM_EMB_DIM floats) from Engine std::vector e(NGM_EMB_DIM, 0.f); e[0]=0.2f; e[1]=0.7f; e[2]=0.1f; // whatever the engine outputs auto pack = ngm.wrapWord("hello", e); std::cout << pack.formatted << "\n"; }catch(const std::exception& e){ std::fprintf(stderr, "ERR: %s\n", e.what()); return 1; } return 0; } #endif