From ff1dd8c57ed84f068cc7f56cd12851d5cbedc95e Mon Sep 17 00:00:00 2001 From: enowdev Date: Sat, 31 Oct 2026 02:01:44 +0700 Subject: [PATCH 1/2] feat: built-in SQLite store, chunks by code structure, secrets never indexed A store that runs inside the binary (vector_store: sqlite): vectors in one SQLite file searched by cosine, FTS5 for hybrid search, so enowx-rag needs nothing else to run, and enowX can embed it as a library. Chunks are cut at top-level declarations, each with its lines and symbol, so a result points at file:line. A git repository is indexed from git ls-files, its ignores honored; .env files, keys, certificates and chunks holding a token are never sent to an embedder. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 21 ++ README.md | 2 +- mcp-server/cmd/mcp-server/main.go | 3 + mcp-server/pkg/config/config.go | 34 +- mcp-server/pkg/indexer/chunks.go | 191 ++++++++++ mcp-server/pkg/indexer/chunks_test.go | 105 ++++++ mcp-server/pkg/indexer/indexer.go | 129 ++++--- mcp-server/pkg/indexer/indexer_test.go | 4 +- mcp-server/pkg/rag/sqlite.go | 470 +++++++++++++++++++++++++ mcp-server/pkg/rag/sqlite_test.go | 81 +++++ mcp-server/pkg/ragbuild/ragbuild.go | 16 +- 11 files changed, 984 insertions(+), 72 deletions(-) create mode 100644 mcp-server/pkg/indexer/chunks.go create mode 100644 mcp-server/pkg/indexer/chunks_test.go create mode 100644 mcp-server/pkg/rag/sqlite.go create mode 100644 mcp-server/pkg/rag/sqlite_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b4cbf5..cef4bb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- **Built-in store (`vector_store: sqlite`)**: one SQLite file (pure Go, no + cgo; `sqlite_path`, default `~/.enowx-rag/rag.db`, or `RAG_SQLITE_PATH`), + vectors searched by cosine, text in FTS5 for hybrid search (RRF), so + enowx-rag runs with nothing else to install. Fits projects of up to a few + hundred thousand chunks. +- **Chunks cut by code structure**: at top-level declarations (Go, Python, + JS/TS, Rust, Java/Kotlin/Swift/C#/PHP, Ruby, Markdown headings, SQL, + protobuf), the comments above each kept with it, small ones packed + together; each chunk carries `start_line`, `end_line` and `symbol`, and + starts `File: path:start-end (symbol)` (`chunk_version` v3, so existing + indexes re-embed once). +- `Indexer.Files` lets a program embedding enowx-rag pass the files to index. + +### Security +- Secrets are never indexed: `.env*`, keys and certificates, `credentials` + and `*secret*` files by name, and any chunk holding a private key block or a + cloud or API token. + +### Changed +- A git repository is indexed from `git ls-files` (tracked and untracked, + `.gitignore` honored) instead of a walk with a fixed ignore list. - **One-line install + prebuilt binaries**: releases now ship cross-compiled, CGO-free binaries for macOS/Linux (amd64/arm64) and Windows (amd64) via GoReleaser (`.github/workflows/release.yml` on tag `v*`). Install without a diff --git a/README.md b/README.md index 2b1d5d4..2e26c29 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ without you re-explaining them every time. - **Persistent memory, durable metrics.** Design decisions and facts survive across sessions and restarts; query latency (p50/p95), token usage, and retrieval breakdown are persisted to a local SQLite DB (pure-Go, no cgo). -- **Bring your own stack.** Vector stores: **Qdrant, pgvector, Chroma**. +- **Bring your own stack.** Vector stores: **a built-in SQLite file** (nothing to install), **Qdrant, pgvector, Chroma**. Embedders: **Voyage AI, any OpenAI-compatible `/v1/embeddings` API, or self-hosted TEI**. Swap models or move stores later with the built-in **Migration** tool (re-embeds stored text — no re-scraping your codebase). diff --git a/mcp-server/cmd/mcp-server/main.go b/mcp-server/cmd/mcp-server/main.go index e7ab0af..80c7e09 100644 --- a/mcp-server/cmd/mcp-server/main.go +++ b/mcp-server/cmd/mcp-server/main.go @@ -57,6 +57,7 @@ type RuntimeConfig struct { QdrantAPIKey string ChromaURL string PGVectorDSN string + SQLitePath string TEIBaseURL string VoyageAPIKey string VoyageModel string @@ -84,6 +85,7 @@ func resolveConfig() (*RuntimeConfig, error) { QdrantAPIKey: cfg.QdrantAPIKey, ChromaURL: cfg.ChromaURL, PGVectorDSN: cfg.PGVectorDSN, + SQLitePath: cfg.StorePath(), TEIBaseURL: cfg.TEIURL, VoyageAPIKey: cfg.Voyage.APIKey, VoyageModel: cfg.Voyage.Model, @@ -112,6 +114,7 @@ func buildProvider(ctx context.Context, cfg *RuntimeConfig) (rag.Provider, error QdrantAPIKey: cfg.QdrantAPIKey, ChromaURL: cfg.ChromaURL, PGVectorDSN: cfg.PGVectorDSN, + SQLitePath: cfg.SQLitePath, VoyageAPIKey: cfg.VoyageAPIKey, VoyageModel: cfg.VoyageModel, VoyageDim: cfg.VectorDim, diff --git a/mcp-server/pkg/config/config.go b/mcp-server/pkg/config/config.go index 1b8854e..650b88b 100644 --- a/mcp-server/pkg/config/config.go +++ b/mcp-server/pkg/config/config.go @@ -36,16 +36,19 @@ type OpenAIConfig struct { // Config holds all configurable settings for the RAG server. Fields use // yaml struct tags so the struct can be marshalled/unmarshalled directly. type Config struct { - VectorStore string `yaml:"vector_store"` - Embedder string `yaml:"embedder"` - Voyage VoyageConfig `yaml:"voyage"` - OpenAI OpenAIConfig `yaml:"openai"` - PGVectorDSN string `yaml:"pgvector_dsn"` - QdrantURL string `yaml:"qdrant_url"` - QdrantAPIKey string `yaml:"qdrant_api_key"` - ChromaURL string `yaml:"chroma_url"` - TEIURL string `yaml:"tei_url"` - RerankerModel string `yaml:"reranker_model"` + VectorStore string `yaml:"vector_store"` + Embedder string `yaml:"embedder"` + Voyage VoyageConfig `yaml:"voyage"` + OpenAI OpenAIConfig `yaml:"openai"` + PGVectorDSN string `yaml:"pgvector_dsn"` + // SQLitePath is the built-in store's file (vector_store: sqlite); + // empty means ~/.enowx-rag/rag.db. + SQLitePath string `yaml:"sqlite_path,omitempty"` + QdrantURL string `yaml:"qdrant_url"` + QdrantAPIKey string `yaml:"qdrant_api_key"` + ChromaURL string `yaml:"chroma_url"` + TEIURL string `yaml:"tei_url"` + RerankerModel string `yaml:"reranker_model"` // AdminToken, when set, gates /api and /mcp with a bearer token. The env var // RAG_ADMIN_TOKEN takes precedence over this file value (see EffectiveAdminToken). AdminToken string `yaml:"admin_token,omitempty"` @@ -91,6 +94,14 @@ func Path() string { return filepath.Join(home, ".enowx-rag", "config.yaml") } +// StorePath is the built-in store's file: cfg's, else ~/.enowx-rag/rag.db. +func (cfg *Config) StorePath() string { + if cfg.SQLitePath != "" { + return cfg.SQLitePath + } + return filepath.Join(filepath.Dir(Path()), "rag.db") +} + // MetricsDBPath returns the absolute path to the durable metrics database: // ~/.enowx-rag/metrics.db. func MetricsDBPath() string { @@ -192,6 +203,9 @@ func applyEnvOverrides(cfg *Config) { if v := os.Getenv("RAG_PGVECTOR_DSN"); v != "" { cfg.PGVectorDSN = v } + if v := os.Getenv("RAG_SQLITE_PATH"); v != "" { + cfg.SQLitePath = v + } if v := os.Getenv("RAG_TEI_URL"); v != "" { cfg.TEIURL = v } diff --git a/mcp-server/pkg/indexer/chunks.go b/mcp-server/pkg/indexer/chunks.go new file mode 100644 index 0000000..17bcb61 --- /dev/null +++ b/mcp-server/pkg/indexer/chunks.go @@ -0,0 +1,191 @@ +package indexer + +import ( + "bytes" + "os/exec" + "path/filepath" + "regexp" + "strings" +) + +// Chunk is a piece of a file: its text, the lines it spans (1-based, End +// included) and the symbol it starts with ("" when none). +type Chunk struct { + Text string + Start, End int + Symbol string +} + +// declarations find where a top-level declaration starts, by language, +// the name it declares in the last group. +var declarations = map[string]*regexp.Regexp{ + ".go": regexp.MustCompile(`^(?:func|type|var|const)\s+(?:\([^)]*\)\s*)?([A-Za-z_][\w]*)`), + ".py": regexp.MustCompile(`^(?:async\s+)?(?:def|class)\s+([A-Za-z_]\w*)`), + ".js": jsDecl, + ".jsx": jsDecl, + ".ts": jsDecl, + ".tsx": jsDecl, + ".vue": jsDecl, + ".svelte": jsDecl, + ".rs": regexp.MustCompile(`^(?:pub(?:\([\w:]+\))?\s+)?(?:async\s+)?(?:fn|struct|enum|trait|impl|mod|type|const|static)\s+(?:<[^>]*>\s*)?([A-Za-z_]\w*)`), + ".java": classDecl, + ".kt": classDecl, + ".swift": classDecl, + ".cs": classDecl, + ".php": classDecl, + ".rb": regexp.MustCompile(`^(?:def|class|module)\s+([\w.:?!]+)`), + ".md": regexp.MustCompile(`^#{1,3}\s+(.+)`), + ".proto": regexp.MustCompile(`^(?:message|service|enum)\s+(\w+)`), + ".sql": regexp.MustCompile(`(?i)^(?:create|alter)\s+(?:table|view|index|function)\s+(?:if\s+not\s+exists\s+)?([\w.]+)`), +} + +var ( + jsDecl = regexp.MustCompile(`^(?:export\s+)?(?:default\s+)?(?:async\s+)?(?:function\*?|class|interface|type|enum|const|let)\s+([A-Za-z_$][\w$]*)`) + classDecl = regexp.MustCompile(`^(?:(?:public|private|protected|internal|static|final|abstract|open|override|data|sealed|export)\s+)*(?:class|interface|enum|struct|record|fun|func|function|object|protocol|extension)\s+([A-Za-z_]\w*)`) + comment = regexp.MustCompile(`^\s*(?://|#|/\*|\*|--|"""|///)`) +) + +// chunkCode cuts file path's text into chunks of at most maxChars, at its +// top-level declarations (with the comments right above them), several +// small ones together; a declaration too long is cut at blank lines, then +// at lines. A language enowX knows no declarations of is cut at blank +// lines. +func chunkCode(text, path string, maxChars int) []Chunk { + lines := strings.Split(strings.ReplaceAll(text, "\r\n", "\n"), "\n") + decl := declarations[strings.ToLower(filepath.Ext(path))] + // Segments: line spans starting at a declaration (or a blank line when + // the language has none), comments above it pulled in. + type seg struct { + start, end int // 0-based, end excluded + symbol string + } + var segs []seg + cur := seg{} + begin := func(i int, symbol string) { + if i > cur.start { + cur.end = i + segs = append(segs, cur) + } + cur = seg{start: i, symbol: symbol} + } + for i, line := range lines { + if decl != nil { + if m := decl.FindStringSubmatch(line); m != nil { + at := i + for at > cur.start && comment.MatchString(lines[at-1]) { + at-- + } + begin(at, strings.TrimSpace(m[len(m)-1])) + } + } else if strings.TrimSpace(line) == "" && i > cur.start { + begin(i, "") + } + } + cur.end = len(lines) + segs = append(segs, cur) + + size := func(s, e int) int { + n := 0 + for _, l := range lines[s:e] { + n += len(l) + 1 + } + return n + } + var out []Chunk + emit := func(s, e int, symbol string) { + body := strings.Join(lines[s:e], "\n") + if strings.TrimSpace(body) == "" { + return + } + out = append(out, Chunk{Text: body, Start: s + 1, End: e, Symbol: symbol}) + } + // Pack small segments together; cut long ones. + packS, packE, packSym := -1, -1, "" + flush := func() { + if packS >= 0 { + emit(packS, packE, packSym) + packS, packE, packSym = -1, -1, "" + } + } + for _, sg := range segs { + n := size(sg.start, sg.end) + if n > maxChars { + flush() + for _, c := range cutLong(lines, sg.start, sg.end, maxChars) { + emit(c[0], c[1], sg.symbol) + } + continue + } + if packS >= 0 && size(packS, sg.end) > maxChars { + flush() + } + if packS < 0 { + packS, packSym = sg.start, sg.symbol + } else if packSym == "" { + packSym = sg.symbol + } + packE = sg.end + } + flush() + return out +} + +// cutLong cuts lines[s:e] into spans of at most maxChars, at blank lines +// when it can, else at a line; a single line longer is cut alone. +func cutLong(lines []string, s, e, maxChars int) [][2]int { + var out [][2]int + start, n, lastBlank := s, 0, -1 + for i := s; i < e; i++ { + l := len(lines[i]) + 1 + if n+l > maxChars && i > start { + cut := i + if lastBlank > start { + cut = lastBlank + } + out = append(out, [2]int{start, cut}) + start, n, lastBlank = cut, size(lines[cut:i]), -1 + } + n += l + if strings.TrimSpace(lines[i]) == "" { + lastBlank = i + } + } + if start < e { + out = append(out, [2]int{start, e}) + } + return out +} + +func size(lines []string) int { + n := 0 + for _, l := range lines { + n += len(l) + 1 + } + return n +} + +// secretName finds files that hold secrets, never indexed. +var secretName = regexp.MustCompile(`(?i)(^|/)(\.env(\..*)?|\.envrc|\.npmrc|\.netrc|\.pypirc|\.git-credentials|id_(rsa|dsa|ecdsa|ed25519)(\.pub)?|.*\.(pem|key|p12|pfx|jks|keystore|crt|cer|der|kdbx|ovpn)|credentials(\.\w+)?|.*secrets?\.(json|ya?ml|toml|env)|service[-_]account.*\.json)$`) + +// secretText finds text that holds a secret: a private key, a cloud key. +var secretText = regexp.MustCompile(`-----BEGIN [A-Z ]*PRIVATE KEY-----|\bAKIA[0-9A-Z]{16}\b|\bgh[pousr]_[A-Za-z0-9]{30,}\b|\bsk-[A-Za-z0-9_-]{20,}\b|\bxox[abpr]-[A-Za-z0-9-]{10,}\b`) + +// IsSecret reports whether the file at path (relative, with slashes) +// holds secrets by its name. +func IsSecret(path string) bool { return secretName.MatchString(path) } + +// gitFiles are the files git keeps or would keep in dir, its ignores +// honored, relative with slashes; false when dir is no repository. +func gitFiles(dir string) ([]string, bool) { + out, err := exec.Command("git", "-C", dir, "ls-files", "-z", "--cached", "--others", "--exclude-standard").Output() + if err != nil { + return nil, false + } + var files []string + for _, f := range bytes.Split(out, []byte{0}) { + if len(f) > 0 { + files = append(files, string(f)) + } + } + return files, true +} diff --git a/mcp-server/pkg/indexer/chunks_test.go b/mcp-server/pkg/indexer/chunks_test.go new file mode 100644 index 0000000..59a00a5 --- /dev/null +++ b/mcp-server/pkg/indexer/chunks_test.go @@ -0,0 +1,105 @@ +package indexer + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/enowdev/enowx-rag/pkg/rag" +) + +// Code is cut at its declarations, each chunk knowing its lines and the +// symbol it starts with, the comment above a declaration kept with it. +func TestChunkCode(t *testing.T) { + src := "package x\n\nimport \"fmt\"\n\n// Parse reads it.\nfunc Parse(s string) int {\n\treturn len(s)\n}\n\ntype Server struct {\n\tAddr string\n}\n\nfunc (s *Server) Start() {\n\tfmt.Println(s.Addr)\n}\n" + got := chunkCode(src, "x.go", 70) + find := func(sym string) Chunk { + for _, c := range got { + if c.Symbol == sym { + return c + } + } + t.Fatalf("no chunk of %s in %+v", sym, got) + return Chunk{} + } + if parse := find("Parse"); parse.Start != 5 || !strings.HasPrefix(parse.Text, "// Parse reads it.") { + t.Fatalf("Parse: %+v", parse) + } + if start := find("Start"); start.Start != 14 || !strings.HasPrefix(start.Text, "func (s *Server) Start()") { + t.Fatalf("Start: %+v", start) + } + for _, c := range got { + if len(c.Text) > 70 { + t.Fatalf("over the size: %+v", c) + } + } + // Small ones pack together within the size. + if big := chunkCode(src, "x.go", 1000); len(big) != 1 || big[0].Start != 1 { + t.Fatalf("packed: %+v", big) + } + // A language without declarations known: blank lines. + if txt := chunkCode("a\nb\n\nc\nd\n", "notes.txt", 4); len(txt) < 2 { + t.Fatalf("paragraphs: %+v", txt) + } +} + +// recording keeps what it's given to index. +type recording struct { + rag.Provider + mu sync.Mutex + docs []rag.Document +} + +func (r *recording) Index(_ context.Context, _ string, docs []rag.Document) error { + r.mu.Lock() + defer r.mu.Unlock() + r.docs = append(r.docs, docs...) + return nil +} + +func (r *recording) ListPoints(context.Context, string, map[string]string) ([]rag.PointInfo, error) { + return nil, nil +} + +// A repository's ignored files and its secrets are never indexed, nor a +// chunk holding a key. +func TestIndexSkipsIgnoredAndSecrets(t *testing.T) { + dir := t.TempDir() + write := func(name, body string) { + os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o755) + os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644) + } + write(".gitignore", "build/\n*.log\n") + write("main.go", "package main\n\nfunc main() {}\n") + write("build/out.go", "package build\n") + write("debug.log", "x") + write(".env", "TOKEN=abc\n") + write("certs/server.pem", "x") + write("config/keys.go", "package config\n\nconst key = \"AKIAABCDEFGHIJKLMNOP\"\n") + if out, err := exec.Command("git", "-C", dir, "init", "-q").CombinedOutput(); err != nil { + t.Skipf("git: %v %s", err, out) + } + rec := &recording{} + idx := NewIndexer(rec, 1000) + if _, err := idx.IndexProject(context.Background(), "p", dir); err != nil { + t.Fatal(err) + } + var files []string + for _, d := range rec.docs { + files = append(files, d.Meta["source_file"]) + if strings.Contains(d.Content, "AKIA") { + t.Fatal("a chunk with a key was indexed") + } + } + got := strings.Join(files, ",") + if !strings.Contains(got, "main.go") || strings.Contains(got, "build/") || strings.Contains(got, ".env") || strings.Contains(got, ".pem") || strings.Contains(got, ".log") { + t.Fatalf("indexed %s", got) + } + if rec.docs[0].Meta["start_line"] == "" || !strings.HasPrefix(rec.docs[0].Content, "File: ") { + t.Fatalf("lines: %+v", rec.docs[0]) + } +} diff --git a/mcp-server/pkg/indexer/indexer.go b/mcp-server/pkg/indexer/indexer.go index 0ce5d49..1eb429b 100644 --- a/mcp-server/pkg/indexer/indexer.go +++ b/mcp-server/pkg/indexer/indexer.go @@ -1,6 +1,7 @@ package indexer import ( + "bytes" "context" "crypto/sha256" "encoding/hex" @@ -17,6 +18,9 @@ import ( type Indexer struct { provider rag.Provider chunkSize int + // Files, when set, lists the files to index in a folder (relative, + // with slashes) in place of git's (or, outside a repository, a walk). + Files func(root string) ([]string, error) } // NewIndexer creates an indexer with the given provider. @@ -32,20 +36,20 @@ func NewIndexer(provider rag.Provider, chunkSize int) *Indexer { // Default ignore directories. var defaultIgnores = map[string]bool{ - "node_modules": true, - ".git": true, - "vendor": true, - "dist": true, - "build": true, - ".next": true, - ".nuxt": true, - "__pycache__": true, - ".cache": true, - "target": true, - ".idea": true, - ".vscode": true, - ".DS_Store": true, - "coverage": true, + "node_modules": true, + ".git": true, + "vendor": true, + "dist": true, + "build": true, + ".next": true, + ".nuxt": true, + "__pycache__": true, + ".cache": true, + "target": true, + ".idea": true, + ".vscode": true, + ".DS_Store": true, + "coverage": true, ".pytest_cache": true, } @@ -83,68 +87,58 @@ func (idx *Indexer) IndexProject(ctx context.Context, projectID, rootDir string) } } - // Walk the directory and collect files. + // The files: the caller's, else git's (its ignores honored), else a + // walk; never one that holds secrets. var docs []rag.Document var currentFiles []string skipped := 0 - - err = filepath.WalkDir(rootDir, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return nil // skip errors - } - if d.IsDir() { - name := d.Name() - if defaultIgnores[name] { - return filepath.SkipDir - } - return nil - } - // Skip binary/irrelevant files - if !isIndexable(d.Name()) { - return nil + files, err := idx.listFiles(rootDir) + if err != nil { + return nil, err + } + for _, relPath := range files { + if IsSecret(relPath) || !isIndexable(filepath.Base(relPath)) { + continue } - - relPath, err := filepath.Rel(rootDir, path) - if err != nil { - return nil + path := filepath.Join(rootDir, filepath.FromSlash(relPath)) + info, err := os.Stat(path) + if err != nil || info.IsDir() || info.Size() == 0 || info.Size() > 500*1024 { + continue } - relPath = filepath.ToSlash(relPath) currentFiles = append(currentFiles, relPath) - content, err := os.ReadFile(path) - if err != nil { - return nil - } - // Skip empty or very large files (>500KB) - if len(content) == 0 || len(content) > 500*1024 { - return nil + if err != nil || bytes.IndexByte(content, 0) >= 0 { + continue // unreadable, or binary } - - chunks := chunkText(string(content), idx.chunkSize) - for i, chunk := range chunks { + for i, ch := range chunkCode(string(content), relPath, idx.chunkSize) { + if secretText.MatchString(ch.Text) { + continue // a key in the code: never sent to an embedder + } docID := fmt.Sprintf("%s/%s#chunk%d", sourceDir, relPath, i) - contentHash := computeContentHash(chunk) - // Skip re-embedding if the content_hash matches an existing point + contentHash := computeContentHash(ch.Text) if existingHash, ok := existingHashes[docID]; ok && existingHash == contentHash { skipped++ continue } + head := fmt.Sprintf("File: %s:%d-%d", relPath, ch.Start, ch.End) + if ch.Symbol != "" { + head += " (" + ch.Symbol + ")" + } docs = append(docs, rag.Document{ ID: docID, - Content: fmt.Sprintf("File: %s\n\n%s", relPath, chunk), + Content: head + "\n\n" + ch.Text, Meta: map[string]string{ "source_file": relPath, "source_dir": sourceDir, "chunk_index": fmt.Sprintf("%d", i), "content_hash": contentHash, - "chunk_version": "v2", + "chunk_version": "v3", + "start_line": fmt.Sprintf("%d", ch.Start), + "end_line": fmt.Sprintf("%d", ch.End), + "symbol": ch.Symbol, }, }) } - return nil - }) - if err != nil { - return nil, err } // Index new/changed documents. @@ -181,6 +175,33 @@ func (idx *Indexer) IndexProject(ctx context.Context, projectID, rootDir string) }, nil } +// listFiles are the files of rootDir to look at, relative with slashes. +func (idx *Indexer) listFiles(rootDir string) ([]string, error) { + if idx.Files != nil { + return idx.Files(rootDir) + } + if files, ok := gitFiles(rootDir); ok { + return files, nil + } + var files []string + err := filepath.WalkDir(rootDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return nil // skip errors + } + if d.IsDir() { + if defaultIgnores[d.Name()] { + return filepath.SkipDir + } + return nil + } + if rel, err := filepath.Rel(rootDir, path); err == nil { + files = append(files, filepath.ToSlash(rel)) + } + return nil + }) + return files, err +} + // SyncResult holds stats from a project sync. type SyncResult struct { Indexed int `json:"indexed"` @@ -272,7 +293,7 @@ func isIndexable(name string) bool { ".sql": true, ".sh": true, ".bash": true, ".zsh": true, ".yml": true, ".yaml": true, ".toml": true, ".json": true, ".xml": true, ".html": true, ".css": true, ".scss": true, - ".md": true, ".txt": true, ".env": true, ".cfg": true, + ".md": true, ".txt": true, ".cfg": true, ".ini": true, ".conf": true, ".dockerfile": true, ".proto": true, ".graphql": true, ".gql": true, } diff --git a/mcp-server/pkg/indexer/indexer_test.go b/mcp-server/pkg/indexer/indexer_test.go index 76f7b3b..a00c856 100644 --- a/mcp-server/pkg/indexer/indexer_test.go +++ b/mcp-server/pkg/indexer/indexer_test.go @@ -173,8 +173,8 @@ func TestIndexerAddsChunkVersion(t *testing.T) { t.Errorf("doc %d: chunk_version not in metadata", i) continue } - if version != "v2" { - t.Errorf("doc %d: chunk_version = %q, want %q", i, version, "v2") + if version != "v3" { + t.Errorf("doc %d: chunk_version = %q, want %q", i, version, "v3") } } } diff --git a/mcp-server/pkg/rag/sqlite.go b/mcp-server/pkg/rag/sqlite.go new file mode 100644 index 0000000..a90089c --- /dev/null +++ b/mcp-server/pkg/rag/sqlite.go @@ -0,0 +1,470 @@ +package rag + +import ( + "context" + "database/sql" + "encoding/binary" + "encoding/json" + "fmt" + "math" + "os" + "path/filepath" + "sort" + "strings" + "sync" + + "github.com/google/uuid" + _ "modernc.org/sqlite" // pure Go, no cgo +) + +// SQLiteProvider keeps every project's points in one SQLite file, the +// vectors as blobs searched by brute force (cosine) and the text in an FTS5 +// table for hybrid search: a store that runs inside the program, nothing +// to install or keep running. It suits a project of up to a few hundred +// thousand chunks; a project's vectors are held in memory once searched. +type SQLiteProvider struct { + db *sql.DB + embedder EmbeddingClient + + mu sync.Mutex + cache map[string][]cached // project → its points' vectors, normalized +} + +type cached struct { + id string + vec []float32 +} + +// NewSQLiteProvider opens (or makes) the store at path. +func NewSQLiteProvider(path string, embedder EmbeddingClient) (*SQLiteProvider, error) { + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return nil, fmt.Errorf("sqlite: %w", err) + } + db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)") + if err != nil { + return nil, fmt.Errorf("sqlite: open: %w", err) + } + db.SetMaxOpenConns(1) // one writer; reads are fast enough in line + for _, q := range []string{ + `CREATE TABLE IF NOT EXISTS points (project TEXT NOT NULL, id TEXT NOT NULL, content TEXT NOT NULL, meta TEXT NOT NULL, vector BLOB NOT NULL, PRIMARY KEY (project, id))`, + `CREATE VIRTUAL TABLE IF NOT EXISTS points_fts USING fts5(content, project UNINDEXED, id UNINDEXED, tokenize = 'unicode61')`, + `CREATE TABLE IF NOT EXISTS projects (project TEXT PRIMARY KEY)`, + } { + if _, err := db.Exec(q); err != nil { + db.Close() + return nil, fmt.Errorf("sqlite: schema: %w", err) + } + } + return &SQLiteProvider{db: db, embedder: embedder, cache: map[string][]cached{}}, nil +} + +func (p *SQLiteProvider) CreateCollection(ctx context.Context, projectID string) error { + _, err := p.db.ExecContext(ctx, `INSERT OR IGNORE INTO projects(project) VALUES (?)`, projectID) + return err +} + +func (p *SQLiteProvider) DeleteCollection(ctx context.Context, projectID string) error { + tx, err := p.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + for _, q := range []string{`DELETE FROM points WHERE project = ?`, `DELETE FROM points_fts WHERE project = ?`, `DELETE FROM projects WHERE project = ?`} { + if _, err := tx.ExecContext(ctx, q, projectID); err != nil { + return err + } + } + p.forget(projectID) + return tx.Commit() +} + +func (p *SQLiteProvider) forget(projectID string) { + p.mu.Lock() + delete(p.cache, projectID) + p.mu.Unlock() +} + +func (p *SQLiteProvider) Index(ctx context.Context, projectID string, docs []Document) error { + if len(docs) == 0 { + return nil + } + texts := make([]string, len(docs)) + for i, d := range docs { + texts[i] = d.Content + } + vectors, err := p.embedder.Embed(ctx, texts) + if err != nil { + return fmt.Errorf("embed: %w", err) + } + if len(vectors) != len(docs) { + return fmt.Errorf("embedding count mismatch") + } + model := "unknown" + if mn, ok := p.embedder.(ModelNamer); ok { + model = mn.ModelName() + } + tx, err := p.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + if _, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO projects(project) VALUES (?)`, projectID); err != nil { + return err + } + for i, d := range docs { + id := d.ID + if id == "" { + id = uuid.NewString() + } + meta := map[string]string{"embed_model": model, "embed_dim": fmt.Sprint(len(vectors[i])), "doc_id": id} + for k, v := range d.Meta { + meta[k] = v + } + raw, _ := json.Marshal(meta) + if _, err := tx.ExecContext(ctx, `INSERT INTO points(project, id, content, meta, vector) VALUES (?, ?, ?, ?, ?) +ON CONFLICT(project, id) DO UPDATE SET content = excluded.content, meta = excluded.meta, vector = excluded.vector`, + projectID, id, d.Content, string(raw), encodeVector(vectors[i])); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `DELETE FROM points_fts WHERE project = ? AND id = ?`, projectID, id); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `INSERT INTO points_fts(content, project, id) VALUES (?, ?, ?)`, d.Content, projectID, id); err != nil { + return err + } + } + if err := tx.Commit(); err != nil { + return err + } + p.forget(projectID) + return nil +} + +// vectors are project's points' vectors, normalized, read once. +func (p *SQLiteProvider) vectors(ctx context.Context, projectID string) ([]cached, error) { + p.mu.Lock() + if c, ok := p.cache[projectID]; ok { + p.mu.Unlock() + return c, nil + } + p.mu.Unlock() + rows, err := p.db.QueryContext(ctx, `SELECT id, vector FROM points WHERE project = ?`, projectID) + if err != nil { + return nil, err + } + defer rows.Close() + var out []cached + for rows.Next() { + var id string + var blob []byte + if err := rows.Scan(&id, &blob); err != nil { + return nil, err + } + out = append(out, cached{id: id, vec: normalize(decodeVector(blob))}) + } + if err := rows.Err(); err != nil { + return nil, err + } + p.mu.Lock() + p.cache[projectID] = out + p.mu.Unlock() + return out, nil +} + +func (p *SQLiteProvider) queryVector(ctx context.Context, query string) ([]float32, error) { + if qe, ok := p.embedder.(QueryEmbedder); ok { + return qe.EmbedQuery(ctx, query) + } + v, err := p.embedder.Embed(ctx, []string{query}) + if err != nil { + return nil, err + } + if len(v) == 0 { + return nil, fmt.Errorf("no embedding returned") + } + return v[0], nil +} + +// dense is the ids of project's points nearest q, with their cosine. +func (p *SQLiteProvider) dense(ctx context.Context, projectID string, q []float32, limit int) ([]Result, error) { + all, err := p.vectors(ctx, projectID) + if err != nil { + return nil, err + } + q = normalize(q) + scored := make([]Result, 0, len(all)) + for _, c := range all { + if len(c.vec) != len(q) { + continue + } + var dot float64 + for i := range q { + dot += float64(q[i]) * float64(c.vec[i]) + } + scored = append(scored, Result{ID: c.id, Score: dot}) + } + sort.Slice(scored, func(i, j int) bool { return scored[i].Score > scored[j].Score }) + if len(scored) > limit { + scored = scored[:limit] + } + return scored, nil +} + +// fill reads rs' content and metadata. +func (p *SQLiteProvider) fill(ctx context.Context, projectID string, rs []Result) ([]Result, error) { + for i := range rs { + var content, raw string + err := p.db.QueryRowContext(ctx, `SELECT content, meta FROM points WHERE project = ? AND id = ?`, projectID, rs[i].ID).Scan(&content, &raw) + if err != nil { + return nil, err + } + rs[i].Content = content + _ = json.Unmarshal([]byte(raw), &rs[i].Meta) + } + return rs, nil +} + +func (p *SQLiteProvider) SemanticSearch(ctx context.Context, projectID, query string, limit int) ([]Result, error) { + if limit <= 0 { + limit = 5 + } + q, err := p.queryVector(ctx, query) + if err != nil { + return nil, fmt.Errorf("embed query: %w", err) + } + rs, err := p.dense(ctx, projectID, q, limit) + if err != nil { + return nil, err + } + return p.fill(ctx, projectID, rs) +} + +// ftsQuery is query as words FTS5 matches any of, its own syntax quoted. +func ftsQuery(query string) string { + var terms []string + for _, w := range strings.FieldsFunc(query, func(r rune) bool { + return !(r == '_' || r >= '0' && r <= '9' || r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r > 127) + }) { + if len(w) > 1 { + terms = append(terms, `"`+w+`"`) + } + } + return strings.Join(terms, " OR ") +} + +// SemanticSearchHybrid fuses the dense ranking with FTS5's (BM25) by +// Reciprocal Rank Fusion, as pgvector's does. +func (p *SQLiteProvider) SemanticSearchHybrid(ctx context.Context, projectID, query string, limit int) ([]Result, error) { + if limit <= 0 { + limit = 5 + } + q, err := p.queryVector(ctx, query) + if err != nil { + return nil, fmt.Errorf("embed query: %w", err) + } + dense, err := p.dense(ctx, projectID, q, limit*4) + if err != nil { + return nil, err + } + fused := map[string]*Result{} + for i, r := range dense { + fused[r.ID] = &Result{ID: r.ID, Score: 1.0 / float64(60+i+1), InDense: true} + } + if fq := ftsQuery(query); fq != "" { + rows, err := p.db.QueryContext(ctx, `SELECT id FROM points_fts WHERE points_fts MATCH ? AND project = ? ORDER BY bm25(points_fts) LIMIT ?`, fq, projectID, limit*4) + if err == nil { + i := 0 + for rows.Next() { + var id string + if rows.Scan(&id) == nil { + r := fused[id] + if r == nil { + r = &Result{ID: id} + fused[id] = r + } + r.Score += 1.0 / float64(60+i+1) + r.InLexical = true + i++ + } + } + rows.Close() + } + } + out := make([]Result, 0, len(fused)) + for _, r := range fused { + out = append(out, *r) + } + sort.Slice(out, func(i, j int) bool { return out[i].Score > out[j].Score }) + if len(out) > limit { + out = out[:limit] + } + return p.fill(ctx, projectID, out) +} + +func (p *SQLiteProvider) Embed(ctx context.Context, text string) ([]float32, error) { + v, err := p.embedder.Embed(ctx, []string{text}) + if err != nil { + return nil, err + } + if len(v) == 0 { + return nil, fmt.Errorf("no embedding returned") + } + return v[0], nil +} + +func (p *SQLiteProvider) DeletePoints(ctx context.Context, projectID string, pointIDs []string) error { + if len(pointIDs) == 0 { + return nil + } + tx, err := p.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + for _, id := range pointIDs { + if _, err := tx.ExecContext(ctx, `DELETE FROM points WHERE project = ? AND id = ?`, projectID, id); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `DELETE FROM points_fts WHERE project = ? AND id = ?`, projectID, id); err != nil { + return err + } + } + if err := tx.Commit(); err != nil { + return err + } + p.forget(projectID) + return nil +} + +// points reads project's points' ids, metadata and content (cut to max +// when max > 0), those matching metaFilter. +func (p *SQLiteProvider) points(ctx context.Context, projectID string, metaFilter map[string]string, max int) ([]Document, error) { + rows, err := p.db.QueryContext(ctx, `SELECT id, content, meta FROM points WHERE project = ?`, projectID) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Document + for rows.Next() { + var d Document + var raw string + if err := rows.Scan(&d.ID, &d.Content, &raw); err != nil { + return nil, err + } + _ = json.Unmarshal([]byte(raw), &d.Meta) + match := true + for k, v := range metaFilter { + if d.Meta[k] != v { + match = false + } + } + if !match { + continue + } + if max > 0 && len(d.Content) > max { + d.Content = d.Content[:max] + } + out = append(out, d) + } + return out, rows.Err() +} + +func (p *SQLiteProvider) ListPointIDs(ctx context.Context, projectID string, metaFilter map[string]string) ([]string, error) { + docs, err := p.points(ctx, projectID, metaFilter, 1) + if err != nil { + return nil, err + } + ids := make([]string, len(docs)) + for i, d := range docs { + ids[i] = d.ID + } + return ids, nil +} + +func (p *SQLiteProvider) ListPoints(ctx context.Context, projectID string, metaFilter map[string]string) ([]PointInfo, error) { + docs, err := p.points(ctx, projectID, metaFilter, 200) + if err != nil { + return nil, err + } + out := make([]PointInfo, len(docs)) + for i, d := range docs { + out[i] = PointInfo{ID: d.ID, SourceFile: d.Meta["source_file"], ContentHash: d.Meta["content_hash"], ChunkVersion: d.Meta["chunk_version"], + DocID: d.Meta["doc_id"], Content: d.Content, ChunkIndex: d.Meta["chunk_index"]} + } + sort.Slice(out, func(i, j int) bool { + if out[i].SourceFile != out[j].SourceFile { + return out[i].SourceFile < out[j].SourceFile + } + return out[i].ChunkIndex < out[j].ChunkIndex + }) + return out, nil +} + +// ExportPoints returns every point of the project, its content whole. +func (p *SQLiteProvider) ExportPoints(ctx context.Context, projectID string) ([]Document, error) { + return p.points(ctx, projectID, nil, 0) +} + +func (p *SQLiteProvider) CountPoints(ctx context.Context, projectID string) (int, error) { + var n int + err := p.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM points WHERE project = ?`, projectID).Scan(&n) + return n, err +} + +func (p *SQLiteProvider) ListProjectIDs(ctx context.Context) ([]string, error) { + rows, err := p.db.QueryContext(ctx, `SELECT project FROM projects UNION SELECT DISTINCT project FROM points ORDER BY 1`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, err + } + out = append(out, id) + } + return out, rows.Err() +} + +func (p *SQLiteProvider) TokensUsed() int64 { + if tc, ok := p.embedder.(TokenCounter); ok { + return tc.TokensUsed() + } + return 0 +} + +func (p *SQLiteProvider) Close() error { return p.db.Close() } + +func encodeVector(v []float32) []byte { + b := make([]byte, 4*len(v)) + for i, f := range v { + binary.LittleEndian.PutUint32(b[4*i:], math.Float32bits(f)) + } + return b +} + +func decodeVector(b []byte) []float32 { + v := make([]float32, len(b)/4) + for i := range v { + v[i] = math.Float32frombits(binary.LittleEndian.Uint32(b[4*i:])) + } + return v +} + +func normalize(v []float32) []float32 { + var n float64 + for _, f := range v { + n += float64(f) * float64(f) + } + if n == 0 { + return v + } + n = math.Sqrt(n) + out := make([]float32, len(v)) + for i, f := range v { + out[i] = float32(float64(f) / n) + } + return out +} diff --git a/mcp-server/pkg/rag/sqlite_test.go b/mcp-server/pkg/rag/sqlite_test.go new file mode 100644 index 0000000..e215da4 --- /dev/null +++ b/mcp-server/pkg/rag/sqlite_test.go @@ -0,0 +1,81 @@ +package rag + +import ( + "context" + "hash/fnv" + "path/filepath" + "strings" + "testing" +) + +// wordsEmbedder embeds a text as its words hashed into 64 buckets: texts +// sharing words come out near each other, enough to test a store. +type wordsEmbedder struct{} + +func (wordsEmbedder) Embed(_ context.Context, texts []string) ([][]float32, error) { + out := make([][]float32, len(texts)) + for i, t := range texts { + v := make([]float32, 64) + for _, w := range strings.Fields(strings.ToLower(t)) { + h := fnv.New32a() + h.Write([]byte(strings.Trim(w, ".,:()"))) + v[h.Sum32()%64]++ + } + out[i] = v + } + return out, nil +} + +func (wordsEmbedder) VectorSize() int { return 64 } + +// The built-in store keeps points per project, finds them by meaning and, +// hybrid, by words too; lists, exports, deletes and survives a reopen. +func TestSQLiteProvider(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "rag.db") + p, err := NewSQLiteProvider(path, wordsEmbedder{}) + if err != nil { + t.Fatal(err) + } + docs := []Document{ + {ID: "a", Content: "func ParseConfig reads the yaml config file", Meta: map[string]string{"source_file": "config.go", "source_dir": "x"}}, + {ID: "b", Content: "the http server routes requests to handlers", Meta: map[string]string{"source_file": "server.go", "source_dir": "x"}}, + {ID: "c", Content: "migrations move rows between vector stores", Meta: map[string]string{"source_file": "migrate.go", "source_dir": "x"}}, + } + if err := p.Index(ctx, "proj", docs); err != nil { + t.Fatal(err) + } + p.Index(ctx, "other", []Document{{ID: "z", Content: "yaml config file elsewhere"}}) + got, err := p.SemanticSearch(ctx, "proj", "read the config file", 2) + if err != nil || len(got) != 2 || got[0].ID != "a" || got[0].Meta["source_file"] != "config.go" { + t.Fatalf("dense: %+v %v", got, err) + } + hy, err := p.SemanticSearchHybrid(ctx, "proj", "handlers", 3) + if err != nil || len(hy) == 0 || hy[0].ID != "b" || !hy[0].InLexical { + t.Fatalf("hybrid: %+v %v", hy, err) + } + if n, _ := p.CountPoints(ctx, "proj"); n != 3 { + t.Fatalf("count %d", n) + } + if pts, _ := p.ListPoints(ctx, "proj", map[string]string{"source_dir": "x"}); len(pts) != 3 || pts[0].SourceFile != "config.go" { + t.Fatalf("list %+v", pts) + } + if err := p.DeletePoints(ctx, "proj", []string{"a"}); err != nil { + t.Fatal(err) + } + p.Close() + p, err = NewSQLiteProvider(path, wordsEmbedder{}) + if err != nil { + t.Fatal(err) + } + defer p.Close() + if docs, _ := p.ExportPoints(ctx, "proj"); len(docs) != 2 { + t.Fatalf("after a reopen: %d", len(docs)) + } + if ids, _ := p.ListProjectIDs(ctx); len(ids) != 2 { + t.Fatalf("projects %v", ids) + } + if got, _ := p.SemanticSearch(ctx, "proj", "config file", 3); len(got) == 0 || got[0].ID == "a" { + t.Fatalf("deleted still found: %+v", got) + } +} diff --git a/mcp-server/pkg/ragbuild/ragbuild.go b/mcp-server/pkg/ragbuild/ragbuild.go index a1852fd..9cdbc94 100644 --- a/mcp-server/pkg/ragbuild/ragbuild.go +++ b/mcp-server/pkg/ragbuild/ragbuild.go @@ -15,15 +15,16 @@ import ( // Spec fully describes one provider + embedder to build. type Spec struct { - VectorStore string // "qdrant" | "chroma" | "pgvector" + VectorStore string // "sqlite" | "qdrant" | "chroma" | "pgvector" Embedder string // "voyage" | "openai" | "tei" // Vector store connection. - QdrantURL string - QdrantAPIKey string - ChromaURL string - PGVectorDSN string + QdrantURL string + QdrantAPIKey string + ChromaURL string + PGVectorDSN string PGVectorTable string // pgvector table name; defaults to "project_memory" + SQLitePath string // the built-in store's file // Embedder settings. VoyageAPIKey string @@ -64,6 +65,11 @@ func BuildProvider(ctx context.Context, spec Spec) (rag.Provider, error) { return nil, err } switch strings.ToLower(spec.VectorStore) { + case "sqlite": + if spec.SQLitePath == "" { + return nil, fmt.Errorf("sqlite_path is required for the sqlite store") + } + return rag.NewSQLiteProvider(spec.SQLitePath, embedder) case "qdrant": return rag.NewQdrantProvider(ctx, spec.QdrantURL, spec.QdrantAPIKey, embedder) case "chroma": From 026d9f7d956bc9c6db5d0c9387a1f8caf9e7f07c Mon Sep 17 00:00:00 2001 From: enowdev Date: Sat, 31 Oct 2026 02:04:56 +0700 Subject: [PATCH 2/2] fix: module path matches its folder, mcp-server MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit go install …/mcp-server/cmd/mcp-server@latest and importing the packages from another program failed: the module declared github.com/enowdev/enowx-rag while living in mcp-server/. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 +++ mcp-server/cmd/mcp-server/main.go | 14 +++++++------- mcp-server/go.mod | 2 +- mcp-server/pkg/core/service.go | 4 ++-- mcp-server/pkg/core/service_test.go | 2 +- mcp-server/pkg/httpapi/auth.go | 2 +- mcp-server/pkg/httpapi/handlers.go | 4 ++-- mcp-server/pkg/httpapi/handlers_test.go | 4 ++-- mcp-server/pkg/httpapi/server.go | 2 +- mcp-server/pkg/httpapi/setup.go | 4 ++-- mcp-server/pkg/httpapi/setup_config.go | 2 +- mcp-server/pkg/httpapi/setup_install.go | 2 +- mcp-server/pkg/httpapi/setup_migrate.go | 10 +++++----- mcp-server/pkg/httpapi/setup_test.go | 2 +- mcp-server/pkg/indexer/chunks_test.go | 2 +- mcp-server/pkg/indexer/indexer.go | 2 +- mcp-server/pkg/indexer/indexer_test.go | 2 +- mcp-server/pkg/migrate/cloud/cloud.go | 2 +- mcp-server/pkg/migrate/cloud/connectors.go | 2 +- mcp-server/pkg/migrate/migrate.go | 2 +- mcp-server/pkg/migrate/migrate_test.go | 2 +- mcp-server/pkg/ragbuild/ragbuild.go | 2 +- 22 files changed, 38 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cef4bb6..ddba061 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 cloud or API token. ### Changed +- The Go module is `github.com/enowdev/enowx-rag/mcp-server`, as its folder + says, so `go install …/mcp-server/cmd/mcp-server@latest` and importing its + packages from another program work. - A git repository is indexed from `git ls-files` (tracked and untracked, `.gitignore` honored) instead of a walk with a fixed ignore list. - **One-line install + prebuilt binaries**: releases now ship cross-compiled, diff --git a/mcp-server/cmd/mcp-server/main.go b/mcp-server/cmd/mcp-server/main.go index 80c7e09..baebfc0 100644 --- a/mcp-server/cmd/mcp-server/main.go +++ b/mcp-server/cmd/mcp-server/main.go @@ -12,13 +12,13 @@ import ( "runtime/debug" "time" - "github.com/enowdev/enowx-rag/pkg/config" - "github.com/enowdev/enowx-rag/pkg/core" - "github.com/enowdev/enowx-rag/pkg/httpapi" - "github.com/enowdev/enowx-rag/pkg/indexer" - "github.com/enowdev/enowx-rag/pkg/rag" - "github.com/enowdev/enowx-rag/pkg/ragbuild" - "github.com/enowdev/enowx-rag/web" + "github.com/enowdev/enowx-rag/mcp-server/pkg/config" + "github.com/enowdev/enowx-rag/mcp-server/pkg/core" + "github.com/enowdev/enowx-rag/mcp-server/pkg/httpapi" + "github.com/enowdev/enowx-rag/mcp-server/pkg/indexer" + "github.com/enowdev/enowx-rag/mcp-server/pkg/rag" + "github.com/enowdev/enowx-rag/mcp-server/pkg/ragbuild" + "github.com/enowdev/enowx-rag/mcp-server/web" "github.com/modelcontextprotocol/go-sdk/mcp" ) diff --git a/mcp-server/go.mod b/mcp-server/go.mod index ae3da6d..d3994bb 100644 --- a/mcp-server/go.mod +++ b/mcp-server/go.mod @@ -1,4 +1,4 @@ -module github.com/enowdev/enowx-rag +module github.com/enowdev/enowx-rag/mcp-server go 1.26 diff --git a/mcp-server/pkg/core/service.go b/mcp-server/pkg/core/service.go index 4f94bd6..8b4e5e9 100644 --- a/mcp-server/pkg/core/service.go +++ b/mcp-server/pkg/core/service.go @@ -10,8 +10,8 @@ import ( "sync" "time" - "github.com/enowdev/enowx-rag/pkg/indexer" - "github.com/enowdev/enowx-rag/pkg/rag" + "github.com/enowdev/enowx-rag/mcp-server/pkg/indexer" + "github.com/enowdev/enowx-rag/mcp-server/pkg/rag" ) // DefaultK is the default number of final results returned by Search. diff --git a/mcp-server/pkg/core/service_test.go b/mcp-server/pkg/core/service_test.go index c8be77c..c8044ba 100644 --- a/mcp-server/pkg/core/service_test.go +++ b/mcp-server/pkg/core/service_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - "github.com/enowdev/enowx-rag/pkg/rag" + "github.com/enowdev/enowx-rag/mcp-server/pkg/rag" ) // --- Mock Provider --- diff --git a/mcp-server/pkg/httpapi/auth.go b/mcp-server/pkg/httpapi/auth.go index 07272fa..b770dbd 100644 --- a/mcp-server/pkg/httpapi/auth.go +++ b/mcp-server/pkg/httpapi/auth.go @@ -5,7 +5,7 @@ import ( "net" "net/http" - "github.com/enowdev/enowx-rag/pkg/config" + "github.com/enowdev/enowx-rag/mcp-server/pkg/config" ) // AdminTokenMiddleware returns an HTTP middleware that protects /api/* and /mcp diff --git a/mcp-server/pkg/httpapi/handlers.go b/mcp-server/pkg/httpapi/handlers.go index a91f52b..1270cc1 100644 --- a/mcp-server/pkg/httpapi/handlers.go +++ b/mcp-server/pkg/httpapi/handlers.go @@ -5,8 +5,8 @@ import ( "net/http" "strconv" - "github.com/enowdev/enowx-rag/pkg/core" - "github.com/enowdev/enowx-rag/pkg/rag" + "github.com/enowdev/enowx-rag/mcp-server/pkg/core" + "github.com/enowdev/enowx-rag/mcp-server/pkg/rag" "github.com/go-chi/chi/v5" ) diff --git a/mcp-server/pkg/httpapi/handlers_test.go b/mcp-server/pkg/httpapi/handlers_test.go index 05439f1..ff4a192 100644 --- a/mcp-server/pkg/httpapi/handlers_test.go +++ b/mcp-server/pkg/httpapi/handlers_test.go @@ -14,8 +14,8 @@ import ( "testing" "time" - "github.com/enowdev/enowx-rag/pkg/core" - "github.com/enowdev/enowx-rag/pkg/rag" + "github.com/enowdev/enowx-rag/mcp-server/pkg/core" + "github.com/enowdev/enowx-rag/mcp-server/pkg/rag" ) // realHome is the developer's HOME captured before any test mutates the env. diff --git a/mcp-server/pkg/httpapi/server.go b/mcp-server/pkg/httpapi/server.go index 5e4e416..37d60a1 100644 --- a/mcp-server/pkg/httpapi/server.go +++ b/mcp-server/pkg/httpapi/server.go @@ -7,7 +7,7 @@ import ( "io/fs" "net/http" - "github.com/enowdev/enowx-rag/pkg/core" + "github.com/enowdev/enowx-rag/mcp-server/pkg/core" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" ) diff --git a/mcp-server/pkg/httpapi/setup.go b/mcp-server/pkg/httpapi/setup.go index 4b90581..b2d94b2 100644 --- a/mcp-server/pkg/httpapi/setup.go +++ b/mcp-server/pkg/httpapi/setup.go @@ -10,8 +10,8 @@ import ( "strings" "time" - "github.com/enowdev/enowx-rag/pkg/config" - "github.com/enowdev/enowx-rag/pkg/rag" + "github.com/enowdev/enowx-rag/mcp-server/pkg/config" + "github.com/enowdev/enowx-rag/mcp-server/pkg/rag" ) // --- Request / response types for setup wizard endpoints --- diff --git a/mcp-server/pkg/httpapi/setup_config.go b/mcp-server/pkg/httpapi/setup_config.go index fc60143..30f5619 100644 --- a/mcp-server/pkg/httpapi/setup_config.go +++ b/mcp-server/pkg/httpapi/setup_config.go @@ -6,7 +6,7 @@ import ( "encoding/json" "net/http" - "github.com/enowdev/enowx-rag/pkg/config" + "github.com/enowdev/enowx-rag/mcp-server/pkg/config" ) // maskSecret returns a masked view of a secret: first 4 + last 3 chars, middle diff --git a/mcp-server/pkg/httpapi/setup_install.go b/mcp-server/pkg/httpapi/setup_install.go index e48dfb6..8a1e91f 100644 --- a/mcp-server/pkg/httpapi/setup_install.go +++ b/mcp-server/pkg/httpapi/setup_install.go @@ -6,7 +6,7 @@ import ( "net/http" "os" - "github.com/enowdev/enowx-rag/pkg/config" + "github.com/enowdev/enowx-rag/mcp-server/pkg/config" ) // mcpServerEntry builds the command + env for the enowx-rag MCP server from the diff --git a/mcp-server/pkg/httpapi/setup_migrate.go b/mcp-server/pkg/httpapi/setup_migrate.go index a6c06fb..a8f3874 100644 --- a/mcp-server/pkg/httpapi/setup_migrate.go +++ b/mcp-server/pkg/httpapi/setup_migrate.go @@ -6,11 +6,11 @@ import ( "net/http" "time" - "github.com/enowdev/enowx-rag/pkg/core" - "github.com/enowdev/enowx-rag/pkg/migrate" - "github.com/enowdev/enowx-rag/pkg/migrate/cloud" - "github.com/enowdev/enowx-rag/pkg/rag" - "github.com/enowdev/enowx-rag/pkg/ragbuild" + "github.com/enowdev/enowx-rag/mcp-server/pkg/core" + "github.com/enowdev/enowx-rag/mcp-server/pkg/migrate" + "github.com/enowdev/enowx-rag/mcp-server/pkg/migrate/cloud" + "github.com/enowdev/enowx-rag/mcp-server/pkg/rag" + "github.com/enowdev/enowx-rag/mcp-server/pkg/ragbuild" ) // migrateRequest describes a re-embed / move migration. The source is a project diff --git a/mcp-server/pkg/httpapi/setup_test.go b/mcp-server/pkg/httpapi/setup_test.go index 351cfa8..55f6eb7 100644 --- a/mcp-server/pkg/httpapi/setup_test.go +++ b/mcp-server/pkg/httpapi/setup_test.go @@ -10,7 +10,7 @@ import ( "strings" "testing" - "github.com/enowdev/enowx-rag/pkg/core" + "github.com/enowdev/enowx-rag/mcp-server/pkg/core" ) // helperClearRagEnv unsets all RAG_ env vars that could interfere with diff --git a/mcp-server/pkg/indexer/chunks_test.go b/mcp-server/pkg/indexer/chunks_test.go index 59a00a5..e5593c8 100644 --- a/mcp-server/pkg/indexer/chunks_test.go +++ b/mcp-server/pkg/indexer/chunks_test.go @@ -9,7 +9,7 @@ import ( "sync" "testing" - "github.com/enowdev/enowx-rag/pkg/rag" + "github.com/enowdev/enowx-rag/mcp-server/pkg/rag" ) // Code is cut at its declarations, each chunk knowing its lines and the diff --git a/mcp-server/pkg/indexer/indexer.go b/mcp-server/pkg/indexer/indexer.go index 1eb429b..d6a5426 100644 --- a/mcp-server/pkg/indexer/indexer.go +++ b/mcp-server/pkg/indexer/indexer.go @@ -11,7 +11,7 @@ import ( "path/filepath" "strings" - "github.com/enowdev/enowx-rag/pkg/rag" + "github.com/enowdev/enowx-rag/mcp-server/pkg/rag" ) // Indexer scans a project directory and syncs its contents to RAG. diff --git a/mcp-server/pkg/indexer/indexer_test.go b/mcp-server/pkg/indexer/indexer_test.go index a00c856..ddf3e72 100644 --- a/mcp-server/pkg/indexer/indexer_test.go +++ b/mcp-server/pkg/indexer/indexer_test.go @@ -9,7 +9,7 @@ import ( "sync" "testing" - "github.com/enowdev/enowx-rag/pkg/rag" + "github.com/enowdev/enowx-rag/mcp-server/pkg/rag" ) // mockProvider captures documents passed to Index and tracks embed calls. diff --git a/mcp-server/pkg/migrate/cloud/cloud.go b/mcp-server/pkg/migrate/cloud/cloud.go index eee7fd6..116fb1c 100644 --- a/mcp-server/pkg/migrate/cloud/cloud.go +++ b/mcp-server/pkg/migrate/cloud/cloud.go @@ -16,7 +16,7 @@ import ( "fmt" "strings" - "github.com/enowdev/enowx-rag/pkg/rag" + "github.com/enowdev/enowx-rag/mcp-server/pkg/rag" ) // Source identifies an external vector DB and where to read text from. diff --git a/mcp-server/pkg/migrate/cloud/connectors.go b/mcp-server/pkg/migrate/cloud/connectors.go index 7acdad4..f0fea5a 100644 --- a/mcp-server/pkg/migrate/cloud/connectors.go +++ b/mcp-server/pkg/migrate/cloud/connectors.go @@ -10,7 +10,7 @@ import ( "strings" "time" - "github.com/enowdev/enowx-rag/pkg/rag" + "github.com/enowdev/enowx-rag/mcp-server/pkg/rag" ) // EXPERIMENTAL connectors below: built from vendor API docs, mock-tested only. diff --git a/mcp-server/pkg/migrate/migrate.go b/mcp-server/pkg/migrate/migrate.go index adb5aaa..9510413 100644 --- a/mcp-server/pkg/migrate/migrate.go +++ b/mcp-server/pkg/migrate/migrate.go @@ -9,7 +9,7 @@ import ( "context" "fmt" - "github.com/enowdev/enowx-rag/pkg/rag" + "github.com/enowdev/enowx-rag/mcp-server/pkg/rag" ) // Progress reports migration advancement. Total is the number of documents to diff --git a/mcp-server/pkg/migrate/migrate_test.go b/mcp-server/pkg/migrate/migrate_test.go index de085d4..cb1c1a7 100644 --- a/mcp-server/pkg/migrate/migrate_test.go +++ b/mcp-server/pkg/migrate/migrate_test.go @@ -4,7 +4,7 @@ import ( "context" "testing" - "github.com/enowdev/enowx-rag/pkg/rag" + "github.com/enowdev/enowx-rag/mcp-server/pkg/rag" ) // fakeProvider is a minimal in-memory rag.Provider for migration tests. As a diff --git a/mcp-server/pkg/ragbuild/ragbuild.go b/mcp-server/pkg/ragbuild/ragbuild.go index 9cdbc94..3a301db 100644 --- a/mcp-server/pkg/ragbuild/ragbuild.go +++ b/mcp-server/pkg/ragbuild/ragbuild.go @@ -10,7 +10,7 @@ import ( "fmt" "strings" - "github.com/enowdev/enowx-rag/pkg/rag" + "github.com/enowdev/enowx-rag/mcp-server/pkg/rag" ) // Spec fully describes one provider + embedder to build.