perf(indexer): optimize indexing speed with native SIMD hashing, Rayon AST parsing, and integer IDs

3 months ago

ae8514a

Authored

JoeJoeflyn8/14/2026, 3:48:03 PM

Parent0982db2

111 files changed

AGENTS.md+11−0
apps/cli/src/cli.ts+38−17
apps/cli/src/mcp-bridge.ts+0−7
apps/cli/src/resolve-cli.ts+5−5
apps/cli/src/setup-agent.ts+135−0
apps/cli/src/setup-claude.ts+0−68
apps/cli/src/setup-codex.ts+0−73
apps/cli/src/setup-devin.ts+0−67
apps/cli/src/setup-instructions.ts+4−0
apps/cli/src/setup-opencode.ts+0−68
apps/cli/src/setup-windsurf.ts+0−67
apps/cli/src/web-server.ts+0−4
apps/cli/tsup.config.ts+8−12
apps/mcp/src/mcp-core.ts+44−25
apps/web/package.json+0−2
apps/web/src/components/context-blocks.tsx+4−3
apps/web/src/components/graph/GraphClient.tsx+13−12
apps/web/src/components/graph/WorkspaceGraph.tsx+25−17
apps/web/src/components/sidebar.tsx+1−1
apps/web/src/components/status-badge.tsx+4−3
apps/web/src/lib/api.ts+22−15
apps/web/src/lib/pagination.tsx+6−1
apps/web/src/lib/theme.tsx+21−4
apps/web/src/lib/utils.ts+8−6
apps/web/src/routes/documents.tsx+0−1
apps/web/src/routes/memories.tsx+1−1
apps/web/src/routes/query.tsx+15−34
apps/web/src/routes/workspaces/$workspaceId/index.tsx+4−3
apps/web/src/routes/workspaces/index.tsx+4−5
apps/web/src/routes/workspaces/new.tsx+12−4
apps/web/src/server/index.ts+36−24
apps/web/src/server/sqlite.ts+208−132
components.json+0−17
landing/scripts/render-og-animated.mjs+1−1
landing/src/app/components/github-stars.tsx+1−1
landing/src/app/components/interactive-graph.tsx+10−3
landing/src/app/components/particle-text.tsx+16−9
landing/src/app/opengraph-image.tsx+1−1
landing/src/app/page.tsx+2−2
package.json+3−3
packages/config/package.json+0−1
packages/config/src/env.ts+0−35
packages/config/src/load-brain-config.ts+1−0
packages/core/src/embeddings.ts+11−14
packages/core/src/graph.ts+37−38
packages/core/src/local-embedding.ts+14−4
packages/core/src/memory.ts+1−1
packages/core/src/retrieval.ts+74−38
packages/core/src/rrf.ts+4−2
packages/core/src/types.ts+2−5
packages/core/src/vector.ts+0−3
packages/db/src/sqlite/chunk-repository.ts+87−84
packages/db/src/sqlite/database-loader.ts+24−16
packages/db/src/sqlite/document-repository.ts+148−143
packages/db/src/sqlite/embedding-repository.ts+59−16
packages/db/src/sqlite/fts-repository.ts+80−38
packages/db/src/sqlite/graph-edge-ops.ts+108−87
packages/db/src/sqlite/graph-node-ops.ts+198−136
packages/db/src/sqlite/graph-ops-shared.ts+20−6
packages/db/src/sqlite/graph-repository.ts+4−9
packages/db/src/sqlite/graph-traversal-ops.ts+83−63
packages/db/src/sqlite/index.ts+1−0
packages/db/src/sqlite/local-workspace.ts+15−7
packages/db/src/sqlite/memory-repository.ts+57−63
packages/db/src/sqlite/registry-db.ts+19−5
packages/db/src/sqlite/registry-repository.ts+265−181
packages/db/src/sqlite/schema.ts+61−38
packages/db/src/sqlite/secure-storage.ts+5−0
packages/db/src/sqlite/shared-types.ts+37−5
packages/db/src/sqlite/types.ts+88−81
packages/db/src/sqlite/utils.ts+17−4
packages/db/src/sqlite/workspace-db.ts+109−44
packages/db/src/sqlite/workspace-repository.ts+91−148
packages/indexer/package.json+0−7
packages/indexer/src/embed-workspace.ts+5−7
packages/indexer/src/graph-service.ts+4−2
packages/indexer/src/hash.ts+0−5
packages/indexer/src/index-workspace.ts+228−181
packages/indexer/src/language.ts+0−1
packages/indexer/src/languages.ts+30−42
packages/indexer/src/markdown.ts+7−3
packages/indexer/src/parsers/fallback-parser.ts+11−10
packages/indexer/src/parsers/oxc-parser.ts+100−52
packages/indexer/src/parsers/tree-sitter-parser.ts+3−6
packages/indexer/src/retrieval-eval.ts+24−16
packages/indexer/src/scanner.ts+57−41
packages/indexer/src/tree-sitter/configs.ts+1−1
packages/indexer/src/tree-sitter/loader.ts+22−11
packages/indexer/src/tree-sitter/parse.ts+8−9
packages/indexer/src/types.ts+22−1
packages/native/Cargo.lock+429−0
packages/native/Cargo.toml+5−0
packages/native/index.d.ts+53−2
packages/native/index.js+4−5
packages/native/index.linux-x64-gnu.node——
packages/native/src/lib.rs+7−21
packages/native/src/oxc_parser.rs+401−0
packages/native/src/parser.rs+14−12
pnpm-lock.yaml+219−731
tests/embedding-blob.test.ts+1−0
tests/embedding-migration.test.ts+10−3
tests/fixtures/retrieval-eval.json+1−1
tests/fts-retrieval.test.ts+11−36
tests/index-workspace.test.ts+3−3
tests/local-embedding.test.ts+22−13
tests/mcp-tools.test.ts+21−6
tests/parser-registry.test.ts+1−1
tests/registry.test.ts+3−1
tests/setup-instructions.test.ts+40−16
tests/vector-search.test.ts+1−0
tests/workspace-db.test.ts+5−5

giogio / openez-graph

perf(indexer): optimize indexing speed with native SIMD hashing, Rayon AST parsing, and integer IDs