diff --git a/CHANGELOG.md b/CHANGELOG.md index a620d7a0f..14d803570 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/). stanza. `brew` commands that load the cask printed "Calling `postflight` is deprecated! Use `postflight_steps` instead." on every invocation. The quarantine strip now runs through the declarative `postflight_steps` DSL. +- Entity extraction no longer treats numbers, versions, and addresses with no + letter (`0.52`, `127.0.0.1`) or dotted abbreviations (`e.g`, `i.e`) as file + paths, and no longer admits ordinary words written in capitals (`NEVER`, + `ONLY`, `NOTE`, `FIXED`) as acronyms. Such tokens linked unrelated memories + through entity edges. ## [0.2.7] - 2026-09-01 diff --git a/internal/memory/graph/entity.go b/internal/memory/graph/entity.go index 1f6572a0e..f59acd0ff 100644 --- a/internal/memory/graph/entity.go +++ b/internal/memory/graph/entity.go @@ -37,13 +37,16 @@ func ValidEntityMode(mode EntityMode) bool { } } +// filePathPattern matches file paths and dotted names (e.g., ./cmd/root.go, +// /etc/config.yml, example.org). Its matches must also pass looksLikePath. +var filePathPattern = regexp.MustCompile(`(?:^|[\s"'(])([.\w/-]+\.\w{1,10})(?:[\s"'),.]|$)`) + var entityPatterns = []*regexp.Regexp{ // CamelCase identifiers (e.g., MyClass, HttpServer) regexp.MustCompile(`\b([A-Z][a-z]+(?:[A-Z][a-z]+)+)\b`), // ALLCAPS acronyms 2-6 chars (e.g., API, HTTP, gRPC, SQL) regexp.MustCompile(`\b([A-Z]{2,6})\b`), - // File paths (e.g., ./cmd/root.go, /etc/config.yml) - regexp.MustCompile(`(?:^|[\s"'(])([.\w/-]+\.\w{1,10})(?:[\s"'),.]|$)`), + filePathPattern, // URLs regexp.MustCompile(`https?://[^\s"'<>)]+`), // @mentions @@ -102,7 +105,12 @@ var techDictionary = map[string]bool{ "MAGMA": true, "MCP": true, "RLM": true, } -// acronymStopwords filters out common English acronyms that aren't tech entities. +// acronymStopwords filters out all-caps tokens that aren't tech entities: +// short common English words, and ordinary words written in capitals for +// emphasis or as labels (NEVER, ONLY, NOTE, FIXED). In a real store the latter +// were among the most frequent "entities" and linked unrelated memories. +// Words that double as technical names (HEAD, POST, PATH, HOME, REST, DELETE, +// LINE) are deliberately absent. var acronymStopwords = map[string]bool{ "IN": true, "ON": true, "AT": true, "TO": true, "BY": true, "OR": true, "AN": true, "IF": true, "IS": true, "IT": true, @@ -117,6 +125,23 @@ var acronymStopwords = map[string]bool{ "SHE": true, "TWO": true, "USE": true, "BOY": true, "DID": true, "GET": true, "HIM": true, "HIS": true, "LET": true, "PUT": true, "TOP": true, "TOO": true, "ANY": true, + // Ordinary words written in capitals for emphasis or as labels. + "OK": true, "YES": true, "TRUE": true, "FALSE": true, + "ONLY": true, "BOTH": true, "NEVER": true, "ALWAYS": true, "MUST": true, + "STILL": true, "SAME": true, "ALSO": true, "EVERY": true, "OTHER": true, + "OWN": true, "REAL": true, "WORTH": true, "HARD": true, "FULL": true, + "EMPTY": true, "ZERO": true, "KEEP": true, "SKIP": true, "LEFT": true, + "WITH": true, "FROM": true, "THAT": true, "THIS": true, "THAN": true, + "WHAT": true, "WHY": true, "DOES": true, "CANNOT": true, "NEEDS": true, + "BEFORE": true, "AFTER": true, "FIRST": true, "SECOND": true, "THIRD": true, + "FINAL": true, "THREE": true, "FOUR": true, "FIVE": true, "HIGH": true, + "PASS": true, "FAIL": true, "FAILED": true, "FIX": true, "FIXED": true, + "DONE": true, "WRONG": true, "STALE": true, "ARMED": true, "OPEN": true, + "CLOSED": true, "FOUND": true, "SENT": true, "SAFE": true, "CLEAN": true, + "VERIFY": true, "CHECK": true, "REVIEW": true, "DRAFT": true, "PROOF": true, + "NOTE": true, "NOTES": true, "LESSON": true, "RULE": true, "FACT": true, + "FACTS": true, "DEFECT": true, "CAUSE": true, "GOTCHA": true, "TRAP": true, + "BUG": true, "ISSUE": true, "ERROR": true, } // ExtractEntities extracts named entities from text using regex patterns @@ -137,6 +162,9 @@ func ExtractEntities(text string) []string { if acronymStopwords[entity] { continue } + if pat == filePathPattern && !looksLikePath(entity) { + continue + } seen[entity] = true entities = append(entities, entity) } @@ -154,6 +182,22 @@ func ExtractEntities(text string) []string { return entities } +// looksLikePath reports whether a file-path match could name a file or +// domain. The path pattern also matches numbers, versions, and addresses with +// no letter (0.52, 127.0.0.1) and dotted single letters (e.g, i.e, J.Y, p.4); +// in a real store those were thousands of entities that named nothing. +func looksLikePath(s string) bool { + if !strings.ContainsFunc(s, func(r rune) bool { return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') }) { + return false + } + for _, segment := range strings.Split(s, ".") { + if len(segment) > 1 { + return true + } + } + return false +} + // ResolveEntities returns the final entity list for a content/provided pair. func ResolveEntities(content string, provided []string, mode EntityMode) []string { return ResolveEntitiesIndexed(content, provided, mode, nil) diff --git a/internal/memory/graph/entity_test.go b/internal/memory/graph/entity_test.go index 2a8087505..c84764cb3 100644 --- a/internal/memory/graph/entity_test.go +++ b/internal/memory/graph/entity_test.go @@ -399,6 +399,34 @@ func TestResolveEntitiesIndexed_NilIndexEqualsResolveEntities(t *testing.T) { } } +func TestExtractEntities_FilePathsSkipNumbersAndAbbreviations(t *testing.T) { + entities := ExtractEntities("Scores 0.52 and 127.0.0.1, e.g. the J.Y. case on p.4, live in cmd/root.go, llama.cpp and example.org") + has := toSet(entities) + for _, want := range []string{"cmd/root.go", "llama.cpp", "example.org"} { + if !has[want] { + t.Errorf("want path %q, got %v", want, entities) + } + } + for _, noise := range []string{"0.52", "127.0.0.1", "e.g", "J.Y", "p.4"} { + if has[noise] { + t.Errorf("number or abbreviation %q must not be an entity, got %v", noise, entities) + } + } +} + +func TestExtractEntities_EmphasisWordsAreNotAcronyms(t *testing.T) { + entities := ExtractEntities("NEVER skip the API check; ONLY use HTTP. Status: OK, FIXED") + has := toSet(entities) + if !has["API"] || !has["HTTP"] { + t.Errorf("want acronyms API and HTTP, got %v", entities) + } + for _, word := range []string{"NEVER", "ONLY", "OK", "FIXED"} { + if has[word] { + t.Errorf("emphasis word %q must not be an entity, got %v", word, entities) + } + } +} + func equalAsSets(a, b []string) bool { if len(a) != len(b) { return false