An issue keeps appearing randomly across datasets. After QuadTreeTiling some transcripts and cells get assigned a -1 label. Those labels are used later to partition and efficiently train segger.
Only happens sometimes. One way to reproduce it is to save a checkpoint of self.data, load it in a notebook, and rerun multiple times. Sometimes it fails after 1–2 runs, other times only after 80+ iterations.
I attached a screenshot with the tiles, cells (red/green), and transcripts (grey) for a failed tiling (left) and a working one (right). Not sure if anyone has seen this before. Happy about any pointers on what could cause this or how to debug it.
import segger.geometry.quadtree
segger.data.tiling
n = 60
# helper class
class TilingEmpty(QuadTreeTiling):
def __init__(self, tiles):
self._tiles = tiles
for i in range(n):
# prep points
points = segger.data.tiling.points_to_geoseries(node_positions, backend='cuspatial')
# quadtree (note: integration/peerlab branch returns 3 outputs here)
_, quadtree, _ = segger.data.tiling.get_quadtree_index(
points,
50000,
with_bounds=True,
)
# to tiles
tiles = segger.data.tiling.quadtree_to_geoseries(quadtree, backend='geopandas')
# assign labels
TILING = TilingEmpty(tiles)
bd_partitions = TILING.label(
data["bd"]["geometry"]
)
# min
bdmin = bd_partitions.min()
print(f"Iteration {i+1}/{n}, Min label: {bdmin}", end="\r")
if bdmin == -1:
print(f"\nFailed at iteration {i+1}")
tiling_failed = TILING
break
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usersoftware/peerd/krauset/.conda/envs/segger_dev/bin/segger", line 6, in <module>
sys.exit(app())
^^^^^
File "/usersoftware/peerd/krauset/.conda/envs/segger_dev/lib/python3.11/site-packages/cyclopts/core.py", line 1947, in __call__
result = _run_maybe_async_command(command, bound, resolved_backend)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usersoftware/peerd/krauset/.conda/envs/segger_dev/lib/python3.11/site-packages/cyclopts/_run.py", line 50, in _run_maybe_async_command
return command(*bound.args, **bound.kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data1/collab002/sail/projects/ongoing/segger_dev/tools/segger/src/segger/cli/segment.py", line 396, in segment
trainer.fit(model=model, datamodule=datamodule)
File "/usersoftware/peerd/krauset/.conda/envs/segger_dev/lib/python3.11/site-packages/lightning/pytorch/trainer/trainer.py", line 584, in fit
call._call_and_handle_interrupt(
File "/usersoftware/peerd/krauset/.conda/envs/segger_dev/lib/python3.11/site-packages/lightning/pytorch/trainer/call.py", line 49, in _call_and_handle_interrupt
return trainer_fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usersoftware/peerd/krauset/.conda/envs/segger_dev/lib/python3.11/site-packages/lightning/pytorch/trainer/trainer.py", line 630, in _fit_impl
self._run(model, ckpt_path=ckpt_path, weights_only=weights_only)
File "/usersoftware/peerd/krauset/.conda/envs/segger_dev/lib/python3.11/site-packages/lightning/pytorch/trainer/trainer.py", line 1039, in _run
call._call_setup_hook(self) # allow user to set up LightningModule in accelerator environment
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usersoftware/peerd/krauset/.conda/envs/segger_dev/lib/python3.11/site-packages/lightning/pytorch/trainer/call.py", line 108, in _call_setup_hook
_call_lightning_datamodule_hook(trainer, "setup", stage=fn)
File "/usersoftware/peerd/krauset/.conda/envs/segger_dev/lib/python3.11/site-packages/lightning/pytorch/trainer/call.py", line 199, in _call_lightning_datamodule_hook
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/data1/collab002/sail/projects/ongoing/segger_dev/tools/segger/src/segger/data/data_module.py", line 285, in setup
self.fit_dataset = TileFitDataset(
^^^^^^^^^^^^^^^
File "/data1/collab002/sail/projects/ongoing/segger_dev/tools/segger/src/segger/data/tile_dataset.py", line 57, in __init__
super().__init__(data=data, partition=partition, clone=clone)
File "/data1/collab002/sail/projects/ongoing/segger_dev/tools/segger/src/segger/data/partition/dataset.py", line 220, in __init__
self._permute_nodes(partition)
File "/data1/collab002/sail/projects/ongoing/segger_dev/tools/segger/src/segger/data/partition/dataset.py", line 358, in _permute_nodes
) = self._permute_node_labels(labels[node])
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data1/collab002/sail/projects/ongoing/segger_dev/tools/segger/src/segger/data/partition/dataset.py", line 395, in _permute_node_labels
raise RuntimeError(
RuntimeError: N Labels: 624327019, Sample labels: tensor([-1, -1, -1, -1, -1, -1, -1, -1, -1, -1]), Min label: -1, Max label: -1. Original error message: bincount only supports 1-d non-negative integral inputs.
An issue keeps appearing randomly across datasets. After
QuadTreeTilingsome transcripts and cells get assigned a-1label. Those labels are used later to partition and efficiently train segger.Only happens sometimes. One way to reproduce it is to save a checkpoint of
self.data, load it in a notebook, and rerun multiple times. Sometimes it fails after 1–2 runs, other times only after 80+ iterations.I attached a screenshot with the tiles, cells (red/green), and transcripts (grey) for a failed tiling (left) and a working one (right). Not sure if anyone has seen this before. Happy about any pointers on what could cause this or how to debug it.