Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 47 additions & 9 deletions pageindex/local_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,21 +95,59 @@ def _write_manifest(self, docs: dict) -> None:

@contextmanager
def lock(self):
"""Cross-process mutex for check-then-write sequences (name
uniquing before save). fcntl is absent on Windows, where the
pre-existing best-effort behavior stays."""
"""Cross-process and cross-thread mutex for check-then-write
sequences (name uniquing before save). Supports POSIX via fcntl
and Windows via msvcrt."""
try:
import fcntl
has_fcntl = True
except ImportError:
has_fcntl = False

try:
import msvcrt
has_msvcrt = True
except ImportError:
has_msvcrt = False

if not has_fcntl and not has_msvcrt:
yield
return

self._root.mkdir(parents=True, exist_ok=True)
with open(self._root / ".lock", "w") as handle:
fcntl.flock(handle, fcntl.LOCK_EX)
try:
yield
finally:
fcntl.flock(handle, fcntl.LOCK_UN)
lock_path = self._root / ".lock"

if has_fcntl:
with open(lock_path, "w") as handle:
fcntl.flock(handle, fcntl.LOCK_EX)
try:
yield
finally:
fcntl.flock(handle, fcntl.LOCK_UN)
elif has_msvcrt:
if not lock_path.is_file():
try:
with open(lock_path, "a+b") as h:
h.write(b"\0")
except OSError:
pass
with open(lock_path, "r+b") as handle:
import time
while True:
try:
handle.seek(0)
msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
break
except OSError:
time.sleep(0.01)
try:
yield
finally:
handle.seek(0)
try:
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
except OSError:
pass

# ── documents ──
def save_document(self, doc_id: str, meta: dict, tree: list, pages: list) -> None:
Expand Down
7 changes: 5 additions & 2 deletions pageindex/page_index_md.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,11 @@
import os
try:
from .utils import *
except:
from utils import *
except (ImportError, ValueError) as _e:
if "relative import" in str(_e).lower() or "no known parent package" in str(_e).lower():
from utils import *
else:
raise

async def get_node_summary(node, summary_token_threshold=200, model=None):
node_text = node.get('text')
Expand Down
40 changes: 35 additions & 5 deletions tests/test_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -413,7 +413,11 @@ def test_env_not_loaded_from_install_dir(tmp_path, tmp_path_factory):
the cwd tree holds none."""
site = tmp_path / "site"
site.mkdir()
(site / "pageindex").symlink_to(Path(__file__).parent.parent / "pageindex")
try:
(site / "pageindex").symlink_to(Path(__file__).parent.parent / "pageindex")
except OSError:
import shutil
shutil.copytree(Path(__file__).parent.parent / "pageindex", site / "pageindex")
(tmp_path / ".env").write_text("PAGEINDEX_API_KEY=pi-leaked\n")
cwd = tmp_path_factory.mktemp("elsewhere")
(cwd / "app.py").write_text(
Expand Down Expand Up @@ -1074,11 +1078,37 @@ def test_data_file_as_directory_fails_loud(local_client, indexed_doc, tmp_path):


def test_list_documents_skips_unsafe_directory_names(
local_client, indexed_doc, tmp_path
local_client, indexed_doc, tmp_path, monkeypatch
):
bad_dir = tmp_path / "store" / "docs" / "bad\\name"
bad_dir.mkdir()
(bad_dir / "doc.json").write_text("{}")
try:
bad_dir = tmp_path / "store" / "docs" / "bad\\name"
bad_dir.mkdir()
(bad_dir / "doc.json").write_text("{}")
except OSError:
# On Windows, backslashes are path separators and cannot exist in
# filesystem directory names. Inject an unsafe entry via scandir instead.
class FakeEntry:
name = "bad\\name"
def is_dir(self):
return True

real_scandir = os.scandir

class FakeScan:
def __init__(self, path):
self._path = Path(path)
self._real = real_scandir(path)
self._is_docs = (self._path == tmp_path / "store" / "docs")
def __enter__(self):
entries = list(self._real.__enter__())
if self._is_docs:
entries.append(FakeEntry())
return entries
def __exit__(self, *args):
return self._real.__exit__(*args)

monkeypatch.setattr(os, "scandir", FakeScan)

listing = local_client.list_documents()
assert [d["id"] for d in listing["documents"]] == [indexed_doc]

Expand Down