From 2fe8a13c8e05e01b666e3a555f39e0ef32c92576 Mon Sep 17 00:00:00 2001 From: Audrey Tang <20723+audreyt@users.noreply.github.com> Date: Tue, 20 Oct 2026 08:43:16 +0800 Subject: [PATCH] fix(memory): keep numbers and emphasis words out of extracted entities The file-path pattern also matched decimals, versions, and addresses with no letter (0.52, 127.0.0.1) and dotted abbreviations (e.g, i.e), and the acronym pattern admitted ordinary words written in capitals for emphasis or as labels (NEVER, ONLY, NOTE, FIXED). On a real store these were among the most frequent stored entities, and entity edges linked memories that shared nothing but a decimal or an emphasis word. File-path matches now need a letter and a dot-separated segment longer than one character, and the acronym stoplist gains the emphasis words while leaving out words that double as technical names (HEAD, POST, PATH). Validated with make test. --- CHANGELOG.md | 5 +++ internal/memory/graph/entity.go | 50 ++++++++++++++++++++++++++-- internal/memory/graph/entity_test.go | 28 ++++++++++++++++ 3 files changed, 80 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a620d7a0f..14d803570 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/). stanza. `brew` commands that load the cask printed "Calling `postflight` is deprecated! Use `postflight_steps` instead." on every invocation. The quarantine strip now runs through the declarative `postflight_steps` DSL. +- Entity extraction no longer treats numbers, versions, and addresses with no + letter (`0.52`, `127.0.0.1`) or dotted abbreviations (`e.g`, `i.e`) as file + paths, and no longer admits ordinary words written in capitals (`NEVER`, + `ONLY`, `NOTE`, `FIXED`) as acronyms. Such tokens linked unrelated memories + through entity edges. ## [0.2.7] - 2026-09-01 diff --git a/internal/memory/graph/entity.go b/internal/memory/graph/entity.go index 1f6572a0e..f59acd0ff 100644 --- a/internal/memory/graph/entity.go +++ b/internal/memory/graph/entity.go @@ -37,13 +37,16 @@ func ValidEntityMode(mode EntityMode) bool { } } +// filePathPattern matches file paths and dotted names (e.g., ./cmd/root.go, +// /etc/config.yml, example.org). Its matches must also pass looksLikePath. +var filePathPattern = regexp.MustCompile(`(?:^|[\s"'(])([.\w/-]+\.\w{1,10})(?:[\s"'),.]|$)`) + var entityPatterns = []*regexp.Regexp{ // CamelCase identifiers (e.g., MyClass, HttpServer) regexp.MustCompile(`\b([A-Z][a-z]+(?:[A-Z][a-z]+)+)\b`), // ALLCAPS acronyms 2-6 chars (e.g., API, HTTP, gRPC, SQL) regexp.MustCompile(`\b([A-Z]{2,6})\b`), - // File paths (e.g., ./cmd/root.go, /etc/config.yml) - regexp.MustCompile(`(?:^|[\s"'(])([.\w/-]+\.\w{1,10})(?:[\s"'),.]|$)`), + filePathPattern, // URLs regexp.MustCompile(`https?://[^\s"'<>)]+`), // @mentions @@ -102,7 +105,12 @@ var techDictionary = map[string]bool{ "MAGMA": true, "MCP": true, "RLM": true, } -// acronymStopwords filters out common English acronyms that aren't tech entities. +// acronymStopwords filters out all-caps tokens that aren't tech entities: +// short common English words, and ordinary words written in capitals for +// emphasis or as labels (NEVER, ONLY, NOTE, FIXED). In a real store the latter +// were among the most frequent "entities" and linked unrelated memories. +// Words that double as technical names (HEAD, POST, PATH, HOME, REST, DELETE, +// LINE) are deliberately absent. var acronymStopwords = map[string]bool{ "IN": true, "ON": true, "AT": true, "TO": true, "BY": true, "OR": true, "AN": true, "IF": true, "IS": true, "IT": true, @@ -117,6 +125,23 @@ var acronymStopwords = map[string]bool{ "SHE": true, "TWO": true, "USE": true, "BOY": true, "DID": true, "GET": true, "HIM": true, "HIS": true, "LET": true, "PUT": true, "TOP": true, "TOO": true, "ANY": true, + // Ordinary words written in capitals for emphasis or as labels. + "OK": true, "YES": true, "TRUE": true, "FALSE": true, + "ONLY": true, "BOTH": true, "NEVER": true, "ALWAYS": true, "MUST": true, + "STILL": true, "SAME": true, "ALSO": true, "EVERY": true, "OTHER": true, + "OWN": true, "REAL": true, "WORTH": true, "HARD": true, "FULL": true, + "EMPTY": true, "ZERO": true, "KEEP": true, "SKIP": true, "LEFT": true, + "WITH": true, "FROM": true, "THAT": true, "THIS": true, "THAN": true, + "WHAT": true, "WHY": true, "DOES": true, "CANNOT": true, "NEEDS": true, + "BEFORE": true, "AFTER": true, "FIRST": true, "SECOND": true, "THIRD": true, + "FINAL": true, "THREE": true, "FOUR": true, "FIVE": true, "HIGH": true, + "PASS": true, "FAIL": true, "FAILED": true, "FIX": true, "FIXED": true, + "DONE": true, "WRONG": true, "STALE": true, "ARMED": true, "OPEN": true, + "CLOSED": true, "FOUND": true, "SENT": true, "SAFE": true, "CLEAN": true, + "VERIFY": true, "CHECK": true, "REVIEW": true, "DRAFT": true, "PROOF": true, + "NOTE": true, "NOTES": true, "LESSON": true, "RULE": true, "FACT": true, + "FACTS": true, "DEFECT": true, "CAUSE": true, "GOTCHA": true, "TRAP": true, + "BUG": true, "ISSUE": true, "ERROR": true, } // ExtractEntities extracts named entities from text using regex patterns @@ -137,6 +162,9 @@ func ExtractEntities(text string) []string { if acronymStopwords[entity] { continue } + if pat == filePathPattern && !looksLikePath(entity) { + continue + } seen[entity] = true entities = append(entities, entity) } @@ -154,6 +182,22 @@ func ExtractEntities(text string) []string { return entities } +// looksLikePath reports whether a file-path match could name a file or +// domain. The path pattern also matches numbers, versions, and addresses with +// no letter (0.52, 127.0.0.1) and dotted single letters (e.g, i.e, J.Y, p.4); +// in a real store those were thousands of entities that named nothing. +func looksLikePath(s string) bool { + if !strings.ContainsFunc(s, func(r rune) bool { return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') }) { + return false + } + for _, segment := range strings.Split(s, ".") { + if len(segment) > 1 { + return true + } + } + return false +} + // ResolveEntities returns the final entity list for a content/provided pair. func ResolveEntities(content string, provided []string, mode EntityMode) []string { return ResolveEntitiesIndexed(content, provided, mode, nil) diff --git a/internal/memory/graph/entity_test.go b/internal/memory/graph/entity_test.go index 2a8087505..c84764cb3 100644 --- a/internal/memory/graph/entity_test.go +++ b/internal/memory/graph/entity_test.go @@ -399,6 +399,34 @@ func TestResolveEntitiesIndexed_NilIndexEqualsResolveEntities(t *testing.T) { } } +func TestExtractEntities_FilePathsSkipNumbersAndAbbreviations(t *testing.T) { + entities := ExtractEntities("Scores 0.52 and 127.0.0.1, e.g. the J.Y. case on p.4, live in cmd/root.go, llama.cpp and example.org") + has := toSet(entities) + for _, want := range []string{"cmd/root.go", "llama.cpp", "example.org"} { + if !has[want] { + t.Errorf("want path %q, got %v", want, entities) + } + } + for _, noise := range []string{"0.52", "127.0.0.1", "e.g", "J.Y", "p.4"} { + if has[noise] { + t.Errorf("number or abbreviation %q must not be an entity, got %v", noise, entities) + } + } +} + +func TestExtractEntities_EmphasisWordsAreNotAcronyms(t *testing.T) { + entities := ExtractEntities("NEVER skip the API check; ONLY use HTTP. Status: OK, FIXED") + has := toSet(entities) + if !has["API"] || !has["HTTP"] { + t.Errorf("want acronyms API and HTTP, got %v", entities) + } + for _, word := range []string{"NEVER", "ONLY", "OK", "FIXED"} { + if has[word] { + t.Errorf("emphasis word %q must not be an entity, got %v", word, entities) + } + } +} + func equalAsSets(a, b []string) bool { if len(a) != len(b) { return false