feat(indexer): optimize reindexing speed with native Rayon AST parsing, prepared statement batching, and pipeline architecture

2 months ago

9507b73

Authored

JoeJoeflyn8/15/2026, 7:29:36 PM

Parent6e08ac1

101 files changed

apps/cli/src/cli.ts+39−17
apps/cli/src/resolve-cli.ts+6−6
apps/cli/src/setup-agent.ts+135−0
apps/cli/src/setup-instructions.ts+4−0
apps/cli/tsup.config.ts+8−12
apps/mcp/src/mcp-core.ts+54−27
apps/web/package.json+0−2
apps/web/src/components/context-blocks.tsx+4−3
apps/web/src/components/graph/GraphClient.tsx+13−12
apps/web/src/components/graph/WorkspaceGraph.tsx+25−17
apps/web/src/components/sidebar.tsx+1−1
apps/web/src/components/status-badge.tsx+4−3
apps/web/src/lib/api.ts+22−15
apps/web/src/lib/pagination.tsx+6−1
apps/web/src/lib/theme.tsx+21−4
apps/web/src/lib/utils.ts+8−6
apps/web/src/routes/documents.tsx+0−1
apps/web/src/routes/memories.tsx+1−1
apps/web/src/routes/query.tsx+15−34
apps/web/src/routes/workspaces/$workspaceId/index.tsx+4−3
apps/web/src/routes/workspaces/index.tsx+4−5
apps/web/src/routes/workspaces/new.tsx+12−4
apps/web/src/server/index.ts+36−24
apps/web/src/server/sqlite.ts+208−132
docs/superpowers/plans/2026-08-16-reindex-pipeline-optimization.md+56−0
docs/superpowers/specs/2026-08-16-reindex-pipeline-optimization-design.md+91−0
landing/scripts/render-og-animated.mjs+1−1
landing/src/app/components/github-stars.tsx+1−1
landing/src/app/components/interactive-graph.tsx+10−3
landing/src/app/components/particle-text.tsx+16−9
landing/src/app/opengraph-image.tsx+1−1
landing/src/app/page.tsx+2−2
package.json+3−3
packages/config/package.json+0−1
packages/config/src/env.ts+0−35
packages/config/src/load-brain-config.ts+1−0
packages/core/src/embeddings.ts+11−14
packages/core/src/graph.ts+37−38
packages/core/src/local-embedding.ts+14−4
packages/core/src/memory.ts+1−1
packages/core/src/retrieval.ts+74−38
packages/core/src/rrf.ts+4−2
packages/core/src/types.ts+2−5
packages/db/src/sqlite/chunk-repository.ts+87−84
packages/db/src/sqlite/database-loader.ts+24−16
packages/db/src/sqlite/document-repository.ts+148−143
packages/db/src/sqlite/embedding-repository.ts+59−16
packages/db/src/sqlite/fts-repository.ts+81−38
packages/db/src/sqlite/graph-edge-ops.ts+108−87
packages/db/src/sqlite/graph-node-ops.ts+198−136
packages/db/src/sqlite/graph-ops-shared.ts+20−6
packages/db/src/sqlite/graph-repository.ts+4−9
packages/db/src/sqlite/graph-traversal-ops.ts+83−63
packages/db/src/sqlite/index.ts+1−0
packages/db/src/sqlite/local-workspace.ts+15−7
packages/db/src/sqlite/memory-repository.ts+57−63
packages/db/src/sqlite/registry-db.ts+19−5
packages/db/src/sqlite/registry-repository.ts+265−181
packages/db/src/sqlite/schema.ts+61−38
packages/db/src/sqlite/secure-storage.ts+5−0
packages/db/src/sqlite/shared-types.ts+37−5
packages/db/src/sqlite/types.ts+88−81
packages/db/src/sqlite/utils.ts+17−4
packages/db/src/sqlite/workspace-db.ts+109−44
packages/db/src/sqlite/workspace-repository.ts+91−148
packages/indexer/package.json+0−7
packages/indexer/src/embed-workspace.ts+5−7
packages/indexer/src/graph-service.ts+4−2
packages/indexer/src/index-workspace.ts+228−181
packages/indexer/src/languages.ts+30−42
packages/indexer/src/markdown.ts+7−3
packages/indexer/src/parsers/fallback-parser.ts+11−10
packages/indexer/src/parsers/oxc-parser.ts+100−52
packages/indexer/src/parsers/tree-sitter-parser.ts+2−5
packages/indexer/src/retrieval-eval.ts+24−16
packages/indexer/src/scanner.ts+47−45
packages/indexer/src/tree-sitter/loader.ts+22−11
packages/indexer/src/tree-sitter/parse.ts+4−6
packages/indexer/src/types.ts+22−1
packages/native/Cargo.lock+429−0
packages/native/Cargo.toml+5−0
packages/native/index.d.ts+53−2
packages/native/index.js+4−5
packages/native/index.linux-x64-gnu.node——
packages/native/src/lib.rs+7−21
packages/native/src/oxc_parser.rs+401−0
packages/native/src/parser.rs+14−12
pnpm-lock.yaml+219−731
tests/e2e-search.test.ts+2−2
tests/embedding-blob.test.ts+1−0
tests/embedding-migration.test.ts+10−3
tests/fixtures/retrieval-eval.json+1−1
tests/fts-retrieval.test.ts+11−36
tests/index-workspace.test.ts+3−3
tests/local-embedding.test.ts+22−13
tests/mcp-tools.test.ts+21−6
tests/parser-registry.test.ts+1−1
tests/registry.test.ts+3−1
tests/setup-instructions.test.ts+40−16
tests/vector-search.test.ts+1−0
tests/workspace-db.test.ts+5−5

giogio / openez-graph

feat(indexer): optimize reindexing speed with native Rayon AST parsing, prepared statement batching, and pipeline architecture