diff --git a/README.md b/README.md index 0dd2ba428..92dd65330 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,18 @@ Configure other models, streaming, multi-document search, citations, and more. Drop PageIndex tools into the OpenAI Agents SDK, the Claude Agent SDK, or any other framework. +### Indexing Markdown Documents via CLI + +PageIndex also supports indexing Markdown documents directly from the command line: + +```bash +python3 run_pageindex.py --md_path document.md --if-add-node-summary yes +``` + +- `--summary-token-threshold ` (default: `200`): Cost optimization for Markdown indexing. When a node contains fewer tokens than this threshold, its raw text is copied verbatim into `summary` without an LLM call. Set to `0` to force LLM summary generation for all nodes. +- `--if-thinning yes/no` (default: `no`): Whether to apply tree thinning for small Markdown sections. +- `--thinning-threshold ` (default: `5000`): Minimum token threshold when tree thinning is enabled. + # Benchmarks diff --git a/pageindex/page_index_md.py b/pageindex/page_index_md.py index 86ef2a145..184545b15 100644 --- a/pageindex/page_index_md.py +++ b/pageindex/page_index_md.py @@ -8,6 +8,12 @@ from utils import * async def get_node_summary(node, summary_token_threshold=200, model=None): + """Generate summary for a markdown node. + + For nodes whose text length is below summary_token_threshold (default: 200 tokens), + the raw node text is copied verbatim into summary without an LLM call to save costs. + Set summary_token_threshold=0 to force LLM summary generation for all nodes. + """ node_text = node.get('text') num_tokens = count_tokens(node_text, model=model) if num_tokens < summary_token_threshold: diff --git a/run_pageindex.py b/run_pageindex.py index 8e3e1e525..20969b716 100644 --- a/run_pageindex.py +++ b/run_pageindex.py @@ -52,7 +52,7 @@ parser.add_argument('--thinning-threshold', type=int, default=5000, help='Minimum token threshold for thinning (markdown only)') parser.add_argument('--summary-token-threshold', type=int, default=200, - help='Token threshold for generating summaries (markdown only)') + help='Token threshold below which node text is copied verbatim into summary without an LLM call (default: 200, markdown only). Set to 0 to generate summaries for all nodes.') args = parser.parse_args() if args.flash: args.mode = 'flash'