Built-in SQLite store, chunks by code structure, secrets never indexed

CHANGELOG.md

@@ -8,6 +8,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Added
+- **Built-in store (`vector_store: sqlite`)**: one SQLite file (pure Go, no
+  cgo; `sqlite_path`, default `~/.enowx-rag/rag.db`, or `RAG_SQLITE_PATH`),
+  vectors searched by cosine, text in FTS5 for hybrid search (RRF), so
+  enowx-rag runs with nothing else to install. Fits projects of up to a few
+  hundred thousand chunks.
+- **Chunks cut by code structure**: at top-level declarations (Go, Python,
+  JS/TS, Rust, Java/Kotlin/Swift/C#/PHP, Ruby, Markdown headings, SQL,
+  protobuf), the comments above each kept with it, small ones packed
+  together; each chunk carries `start_line`, `end_line` and `symbol`, and
+  starts `File: path:start-end (symbol)` (`chunk_version` v3, so existing
+  indexes re-embed once).
+- `Indexer.Files` lets a program embedding enowx-rag pass the files to index.
+
+### Security
+- Secrets are never indexed: `.env*`, keys and certificates, `credentials`
+  and `*secret*` files by name, and any chunk holding a private key block or a
+  cloud or API token.
+
+### Changed
+- The Go module is `github.com/enowdev/enowx-rag/mcp-server`, as its folder
+  says, so `go install …/mcp-server/cmd/mcp-server@latest` and importing its
+  packages from another program work.
+- A git repository is indexed from `git ls-files` (tracked and untracked,
+  `.gitignore` honored) instead of a walk with a fixed ignore list.
 - **One-line install + prebuilt binaries**: releases now ship cross-compiled,
   CGO-free binaries for macOS/Linux (amd64/arm64) and Windows (amd64) via
   GoReleaser (`.github/workflows/release.yml` on tag `v*`). Install without a

README.md

@@ -55,7 +55,7 @@ without you re-explaining them every time.
 - **Persistent memory, durable metrics.** Design decisions and facts survive
   across sessions and restarts; query latency (p50/p95), token usage, and
   retrieval breakdown are persisted to a local SQLite DB (pure-Go, no cgo).
-- **Bring your own stack.** Vector stores: **Qdrant, pgvector, Chroma**.
+- **Bring your own stack.** Vector stores: **a built-in SQLite file** (nothing to install), **Qdrant, pgvector, Chroma**.
   Embedders: **Voyage AI, any OpenAI-compatible `/v1/embeddings` API, or
   self-hosted TEI**. Swap models or move stores later with the built-in
   **Migration** tool (re-embeds stored text — no re-scraping your codebase).

mcp-server/cmd/mcp-server/main.go

@@ -12,13 +12,13 @@ import (
 	"runtime/debug"
 	"time"
 
-	"github.com/enowdev/enowx-rag/pkg/config"
-	"github.com/enowdev/enowx-rag/pkg/core"
-	"github.com/enowdev/enowx-rag/pkg/httpapi"
-	"github.com/enowdev/enowx-rag/pkg/indexer"
-	"github.com/enowdev/enowx-rag/pkg/rag"
-	"github.com/enowdev/enowx-rag/pkg/ragbuild"
-	"github.com/enowdev/enowx-rag/web"
+	"github.com/enowdev/enowx-rag/mcp-server/pkg/config"
+	"github.com/enowdev/enowx-rag/mcp-server/pkg/core"
+	"github.com/enowdev/enowx-rag/mcp-server/pkg/httpapi"
+	"github.com/enowdev/enowx-rag/mcp-server/pkg/indexer"
+	"github.com/enowdev/enowx-rag/mcp-server/pkg/rag"
+	"github.com/enowdev/enowx-rag/mcp-server/pkg/ragbuild"
+	"github.com/enowdev/enowx-rag/mcp-server/web"
 	"github.com/modelcontextprotocol/go-sdk/mcp"
 )
 
@@ -57,6 +57,7 @@ type RuntimeConfig struct {
 	QdrantAPIKey  string
 	ChromaURL     string
 	PGVectorDSN   string
+	SQLitePath    string
 	TEIBaseURL    string
 	VoyageAPIKey  string
 	VoyageModel   string
@@ -84,6 +85,7 @@ func resolveConfig() (*RuntimeConfig, error) {
 		QdrantAPIKey:  cfg.QdrantAPIKey,
 		ChromaURL:     cfg.ChromaURL,
 		PGVectorDSN:   cfg.PGVectorDSN,
+		SQLitePath:    cfg.StorePath(),
 		TEIBaseURL:    cfg.TEIURL,
 		VoyageAPIKey:  cfg.Voyage.APIKey,
 		VoyageModel:   cfg.Voyage.Model,
@@ -112,6 +114,7 @@ func buildProvider(ctx context.Context, cfg *RuntimeConfig) (rag.Provider, error
 		QdrantAPIKey:  cfg.QdrantAPIKey,
 		ChromaURL:     cfg.ChromaURL,
 		PGVectorDSN:   cfg.PGVectorDSN,
+		SQLitePath:    cfg.SQLitePath,
 		VoyageAPIKey:  cfg.VoyageAPIKey,
 		VoyageModel:   cfg.VoyageModel,
 		VoyageDim:     cfg.VectorDim,

mcp-server/go.mod

@@ -1,4 +1,4 @@
-module github.com/enowdev/enowx-rag
+module github.com/enowdev/enowx-rag/mcp-server
 
 go 1.26
 

mcp-server/pkg/config/config.go

@@ -36,16 +36,19 @@ type OpenAIConfig struct {
 // Config holds all configurable settings for the RAG server. Fields use
 // yaml struct tags so the struct can be marshalled/unmarshalled directly.
 type Config struct {
-	VectorStore   string       `yaml:"vector_store"`
-	Embedder      string       `yaml:"embedder"`
-	Voyage        VoyageConfig `yaml:"voyage"`
-	OpenAI        OpenAIConfig `yaml:"openai"`
-	PGVectorDSN   string       `yaml:"pgvector_dsn"`
-	QdrantURL     string       `yaml:"qdrant_url"`
-	QdrantAPIKey  string       `yaml:"qdrant_api_key"`
-	ChromaURL     string       `yaml:"chroma_url"`
-	TEIURL        string       `yaml:"tei_url"`
-	RerankerModel string       `yaml:"reranker_model"`
+	VectorStore string       `yaml:"vector_store"`
+	Embedder    string       `yaml:"embedder"`
+	Voyage      VoyageConfig `yaml:"voyage"`
+	OpenAI      OpenAIConfig `yaml:"openai"`
+	PGVectorDSN string       `yaml:"pgvector_dsn"`
+	// SQLitePath is the built-in store's file (vector_store: sqlite);
+	// empty means ~/.enowx-rag/rag.db.
+	SQLitePath    string `yaml:"sqlite_path,omitempty"`
+	QdrantURL     string `yaml:"qdrant_url"`
+	QdrantAPIKey  string `yaml:"qdrant_api_key"`
+	ChromaURL     string `yaml:"chroma_url"`
+	TEIURL        string `yaml:"tei_url"`
+	RerankerModel string `yaml:"reranker_model"`
 	// AdminToken, when set, gates /api and /mcp with a bearer token. The env var
 	// RAG_ADMIN_TOKEN takes precedence over this file value (see EffectiveAdminToken).
 	AdminToken string `yaml:"admin_token,omitempty"`
@@ -91,6 +94,14 @@ func Path() string {
 	return filepath.Join(home, ".enowx-rag", "config.yaml")
 }
 
+// StorePath is the built-in store's file: cfg's, else ~/.enowx-rag/rag.db.
+func (cfg *Config) StorePath() string {
+	if cfg.SQLitePath != "" {
+		return cfg.SQLitePath
+	}
+	return filepath.Join(filepath.Dir(Path()), "rag.db")
+}
+
 // MetricsDBPath returns the absolute path to the durable metrics database:
 // ~/.enowx-rag/metrics.db.
 func MetricsDBPath() string {
@@ -192,6 +203,9 @@ func applyEnvOverrides(cfg *Config) {
 	if v := os.Getenv("RAG_PGVECTOR_DSN"); v != "" {
 		cfg.PGVectorDSN = v
 	}
+	if v := os.Getenv("RAG_SQLITE_PATH"); v != "" {
+		cfg.SQLitePath = v
+	}
 	if v := os.Getenv("RAG_TEI_URL"); v != "" {
 		cfg.TEIURL = v
 	}

mcp-server/pkg/core/service.go

@@ -10,8 +10,8 @@ import (
 	"sync"
 	"time"
 
-	"github.com/enowdev/enowx-rag/pkg/indexer"
-	"github.com/enowdev/enowx-rag/pkg/rag"
+	"github.com/enowdev/enowx-rag/mcp-server/pkg/indexer"
+	"github.com/enowdev/enowx-rag/mcp-server/pkg/rag"
 )
 
 // DefaultK is the default number of final results returned by Search.

mcp-server/pkg/core/service_test.go

@@ -10,7 +10,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/enowdev/enowx-rag/pkg/rag"
+	"github.com/enowdev/enowx-rag/mcp-server/pkg/rag"
 )
 
 // --- Mock Provider ---

mcp-server/pkg/httpapi/auth.go

@@ -5,7 +5,7 @@ import (
 	"net"
 	"net/http"
 
-	"github.com/enowdev/enowx-rag/pkg/config"
+	"github.com/enowdev/enowx-rag/mcp-server/pkg/config"
 )
 
 // AdminTokenMiddleware returns an HTTP middleware that protects /api/* and /mcp

mcp-server/pkg/httpapi/handlers.go

@@ -5,8 +5,8 @@ import (
 	"net/http"
 	"strconv"
 
-	"github.com/enowdev/enowx-rag/pkg/core"
-	"github.com/enowdev/enowx-rag/pkg/rag"
+	"github.com/enowdev/enowx-rag/mcp-server/pkg/core"
+	"github.com/enowdev/enowx-rag/mcp-server/pkg/rag"
 	"github.com/go-chi/chi/v5"
 )
 

mcp-server/pkg/httpapi/handlers_test.go

@@ -14,8 +14,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/enowdev/enowx-rag/pkg/core"
-	"github.com/enowdev/enowx-rag/pkg/rag"
+	"github.com/enowdev/enowx-rag/mcp-server/pkg/core"
+	"github.com/enowdev/enowx-rag/mcp-server/pkg/rag"
 )
 
 // realHome is the developer's HOME captured before any test mutates the env.

mcp-server/pkg/httpapi/server.go

@@ -7,7 +7,7 @@ import (
 	"io/fs"
 	"net/http"
 
-	"github.com/enowdev/enowx-rag/pkg/core"
+	"github.com/enowdev/enowx-rag/mcp-server/pkg/core"
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/chi/v5/middleware"
 )

mcp-server/pkg/httpapi/setup.go

@@ -10,8 +10,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/enowdev/enowx-rag/pkg/config"
-	"github.com/enowdev/enowx-rag/pkg/rag"
+	"github.com/enowdev/enowx-rag/mcp-server/pkg/config"
+	"github.com/enowdev/enowx-rag/mcp-server/pkg/rag"
 )
 
 // --- Request / response types for setup wizard endpoints ---

mcp-server/pkg/httpapi/setup_config.go

@@ -6,7 +6,7 @@ import (
 	"encoding/json"
 	"net/http"
 
-	"github.com/enowdev/enowx-rag/pkg/config"
+	"github.com/enowdev/enowx-rag/mcp-server/pkg/config"
 )
 
 // maskSecret returns a masked view of a secret: first 4 + last 3 chars, middle

mcp-server/pkg/httpapi/setup_install.go

@@ -6,7 +6,7 @@ import (
 	"net/http"
 	"os"
 
-	"github.com/enowdev/enowx-rag/pkg/config"
+	"github.com/enowdev/enowx-rag/mcp-server/pkg/config"
 )
 
 // mcpServerEntry builds the command + env for the enowx-rag MCP server from the

mcp-server/pkg/httpapi/setup_migrate.go

@@ -6,11 +6,11 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/enowdev/enowx-rag/pkg/core"
-	"github.com/enowdev/enowx-rag/pkg/migrate"
-	"github.com/enowdev/enowx-rag/pkg/migrate/cloud"
-	"github.com/enowdev/enowx-rag/pkg/rag"
-	"github.com/enowdev/enowx-rag/pkg/ragbuild"
+	"github.com/enowdev/enowx-rag/mcp-server/pkg/core"
+	"github.com/enowdev/enowx-rag/mcp-server/pkg/migrate"
+	"github.com/enowdev/enowx-rag/mcp-server/pkg/migrate/cloud"
+	"github.com/enowdev/enowx-rag/mcp-server/pkg/rag"
+	"github.com/enowdev/enowx-rag/mcp-server/pkg/ragbuild"
 )
 
 // migrateRequest describes a re-embed / move migration. The source is a project

mcp-server/pkg/httpapi/setup_test.go

@@ -10,7 +10,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/enowdev/enowx-rag/pkg/core"
+	"github.com/enowdev/enowx-rag/mcp-server/pkg/core"
 )
 
 // helperClearRagEnv unsets all RAG_ env vars that could interfere with