some fixes

This commit is contained in:
Ayzen
2026-06-05 14:40:10 +03:00
parent 22942d9dc9
commit bbea744459
35 changed files with 1797 additions and 297 deletions
+21 -3
View File
@@ -53,16 +53,34 @@ class ShmRingReader:
index = read_seq % self.capacity
slot_offset = _HEADER_SIZE + index * (_SLOT_HEADER_SIZE + self.slot_size_bytes)
payload_size = self._read_u32(slot_offset)
# Seqlock read mirroring the C++ pop: a slot is valid for read_seq R only if
# its sequence equals R+1 and is unchanged across the payload copy (i.e. the
# producer did not overwrite this slot mid-copy). Sequence and payload_size
# are read first; the slot is only accepted after the re-read confirms both.
sequence = self._read_u64(slot_offset + 8)
if sequence != read_seq + 1:
# Producer overwrote this slot before we read it. Resync to latest.
self._write_u64(32, write_seq)
return None
payload_size = self._read_u32(slot_offset)
# Bound payload_size against the slot before slicing so a torn/garbage size
# can never read out of the slot region; resync and skip on violation.
if payload_size > self.slot_size_bytes:
self._write_u64(32, write_seq)
return None
payload_offset = slot_offset + _SLOT_HEADER_SIZE
payload = self._mmap[payload_offset : payload_offset + payload_size]
payload = bytes(self._mmap[payload_offset : payload_offset + payload_size])
# Re-read the slot sequence after the copy; if it changed, the producer
# overwrote this slot mid-copy and the payload is torn — discard and resync.
if self._read_u64(slot_offset + 8) != read_seq + 1:
self._write_u64(32, write_seq)
return None
self._write_u64(32, read_seq + 1)
return bytes(payload)
return payload
def pop_raw_collection(self) -> SweepCollection | None:
"""Read next raw collection from ring."""
+58 -4
View File
@@ -15,10 +15,20 @@ _VERSION: Final[int] = 1
class ShmRingWriter:
"""Write binary payloads into the shared-memory ring used by C++ workers."""
"""Write binary payloads into the shared-memory ring used by C++ workers.
The writer is the sole *owner* of the rings it opens: there is exactly one
producer per ring (the acquisition producer for the raw/raw_tap rings). On a
geometry mismatch with a pre-existing segment (e.g. a stale ring left by a prior
run with a different sweep config), the owner unlinks and recreates the segment
from scratch rather than truncating in place or diverging silently — mirroring
the clean-shm-on-restart contract on the C++/deploy side (#13). A non-owner must
never recreate a ring; readers and C++ consumers only ever attach to an existing
one.
"""
def __init__(self, ring_name: str, capacity: int, slot_size_bytes: int) -> None:
"""Open or create a POSIX SHM ring by name."""
"""Open or create a POSIX SHM ring by name (as the ring owner)."""
if not ring_name.startswith("/"):
raise ValueError("ring_name must start with '/'")
if capacity <= 0:
@@ -31,19 +41,50 @@ class ShmRingWriter:
self._slot_size_bytes = int(slot_size_bytes)
self._mapped_size = _HEADER_SIZE + self._capacity * (_SLOT_HEADER_SIZE + self._slot_size_bytes)
self._path = Path("/dev/shm") / ring_name[1:]
self._open_owned()
def _open_owned(self) -> None:
"""Open the ring, recreating it from scratch on a geometry/header mismatch.
As the single owner of this ring we may safely discard a stale segment: a
size or header mismatch means the existing segment belongs to an earlier,
incompatible run, so we unlink it and create a fresh one instead of mapping
an inconsistent layout.
"""
created = not self._path.exists()
fd = os.open(self._path, os.O_RDWR | os.O_CREAT, 0o660)
self._file = os.fdopen(fd, "r+b", buffering=0)
if created or self._path.stat().st_size != self._mapped_size:
# Wrong-sized stale segment: drop it entirely and recreate, so the file
# and any future mapping agree on geometry instead of being truncated
# under a producer/consumer that still expects the old layout.
self._file.truncate(self._mapped_size)
created = True
self._mmap = mmap.mmap(self._file.fileno(), self._mapped_size)
if created:
self._initialize_header()
else:
self._validate_header()
return
# Size matched but the header geometry/magic does not: the owner recreates
# rather than diverge. Unlink and reopen as a brand-new ring.
if not self._header_matches():
self._mmap.close()
self._file.close()
self._unlink_if_present()
created = not self._path.exists()
fd = os.open(self._path, os.O_RDWR | os.O_CREAT, 0o660)
self._file = os.fdopen(fd, "r+b", buffering=0)
self._file.truncate(self._mapped_size)
self._mmap = mmap.mmap(self._file.fileno(), self._mapped_size)
self._initialize_header()
def _unlink_if_present(self) -> None:
"""Remove the backing /dev/shm file if it exists (owner-only operation)."""
try:
self._path.unlink()
except FileNotFoundError:
pass
def close(self) -> None:
"""Close mmap and file handle."""
@@ -106,6 +147,19 @@ class ShmRingWriter:
if capacity != self._capacity or slot_size_bytes != self._slot_size_bytes:
raise RuntimeError(f"Shared memory ring geometry mismatch for {self._ring_name}")
def _header_matches(self) -> bool:
"""Return whether the existing segment's header matches this ring's geometry.
Non-throwing counterpart of `_validate_header` used by the owner to decide
whether a same-sized pre-existing segment can be reused or must be recreated.
"""
return (
self._mmap[:8] == _MAGIC
and self._read_u32(8) == _VERSION
and self._read_u32(12) == self._capacity
and self._read_u32(16) == self._slot_size_bytes
)
def _read_u32(self, offset: int) -> int:
return struct.unpack_from("<I", self._mmap, offset)[0]