diff --git a/docs/16BIT_TIFF_SUPPORT.md b/docs/16BIT_TIFF_SUPPORT.md index 97ae713..adf703d 100644 --- a/docs/16BIT_TIFF_SUPPORT.md +++ b/docs/16BIT_TIFF_SUPPORT.md @@ -2,17 +2,18 @@ ## Overview -This document describes the implementation of 16-bit grayscale TIFF support for YOLO object detection. The system properly loads 16-bit TIFF images, normalizes them to float32 [0-1], and passes them directly to YOLO **without uint8 conversion** to preserve the full dynamic range and avoid data loss. +This document describes the implementation of 16-bit grayscale TIFF support for YOLO object detection. The system properly loads 16-bit TIFF images, normalizes them to float32 [0-1], and handles them appropriately for both **inference** and **training** **without uint8 conversion** to preserve the full dynamic range and avoid data loss. ## Key Features -✅ Reads 16-bit or float32 images using tifffile -✅ Converts to float32 [0-1] (NO uint8 conversion) -✅ Replicates grayscale → RGB (3 channels) -✅ Passes numpy arrays directly to YOLO (no file I/O) -✅ Uses Ultralytics YOLOv8/v11 models -✅ Works with segmentation models -✅ No data loss, no double normalization, no silent clipping +✅ Reads 16-bit or float32 images using tifffile +✅ Converts to float32 [0-1] (NO uint8 conversion) +✅ Replicates grayscale → RGB (3 channels) +✅ **Inference**: Passes numpy arrays directly to YOLO (no file I/O) +✅ **Training**: Creates float32 3-channel TIFF dataset cache +✅ Uses Ultralytics YOLOv8/v11 models +✅ Works with segmentation models +✅ No data loss, no double normalization, no silent clipping ## Changes Made @@ -46,7 +47,9 @@ Enhanced [`YOLOWrapper._prepare_source()`](../src/model/yolo_wrapper.py:231) to: ## Processing Pipeline -For 16-bit TIFF files: +### For Inference (predict) + +For 16-bit TIFF files during inference: 1. **Load**: File loaded using `tifffile` → preserves 16-bit uint16 data 2. **Normalize**: Convert to float32 and scale to [0, 1] @@ -60,12 +63,28 @@ For 16-bit TIFF files: 4. **Pass to YOLO**: Return float32 array directly (no uint8, no file I/O) 5. **Inference**: YOLO processes the float32 [0-1] RGB array +### For Training (train) + +During training, YOLO's internal dataloader loads images from disk, so we create a cached 3-channel dataset: + +1. **Detect**: Check if dataset contains 16-bit TIFF files +2. **Create Cache**: Build float32 3-channel TIFF dataset in `data/datasets/_float32_cache/` +3. **Convert Each Image**: + - Load 16-bit TIFF using `tifffile` + - Normalize to float32 [0-1] + - Replicate to 3 channels + - Save as float32 TIFF (preserves precision) +4. **Copy Labels**: Copy label files unchanged +5. **Generate data.yaml**: Points to cached 3-channel dataset +6. **Train**: YOLO trains on float32 3-channel TIFFs + ### No Data Loss! -Unlike the previous approach that converted to uint8 (256 levels), the new implementation: +Unlike approaches that convert to uint8 (256 levels), this implementation: - Preserves full 16-bit dynamic range (65536 levels) - Maintains precision with float32 representation -- Passes data directly without intermediate file conversions +- For inference: passes data directly without file conversions +- For training: uses float32 TIFFs (not uint8 PNGs) ## Usage @@ -188,14 +207,16 @@ This test shows the old behavior (uint8 conversion) - kept for comparison. For a 2048×2048 single-channel image: -| Format | Memory | Notes | -|--------|--------|-------| -| Original 16-bit | 8 MB | uint16 grayscale | -| Float32 grayscale | 16 MB | Intermediate | -| Float32 RGB | 48 MB | Final (3 channels) | -| uint8 RGB (old) | 12 MB | OLD approach with data loss | +| Format | Memory | Disk Space | Notes | +|--------|--------|------------|-------| +| Original 16-bit | 8 MB | ~8 MB | uint16 grayscale TIFF | +| Float32 grayscale | 16 MB | - | Intermediate | +| Float32 3-channel | 48 MB | ~48 MB | Training cache | +| uint8 RGB (old) | 12 MB | ~12 MB | OLD approach with data loss | -The float32 approach uses ~4× more memory than uint8 but preserves **all information**. +The float32 approach uses ~4× more memory and disk space than uint8 but preserves **all information**. + +**Cache Directory**: Training creates cached datasets in `data/datasets/_float32_cache/_/` ### Why Direct Numpy Array? diff --git a/src/gui/tabs/training_tab.py b/src/gui/tabs/training_tab.py index 67855cb..0e0f8cd 100644 --- a/src/gui/tabs/training_tab.py +++ b/src/gui/tabs/training_tab.py @@ -3,11 +3,14 @@ Training tab for the microscopy object detection application. Handles model training with YOLO. """ +import hashlib import shutil from datetime import datetime from pathlib import Path from typing import Any, Dict, List, Optional, Tuple +import numpy as np +import tifffile import yaml from PySide6.QtCore import Qt, QThread, Signal from PySide6.QtWidgets import ( @@ -946,6 +949,9 @@ class TrainingTab(QWidget): for msg in split_messages: self._append_training_log(msg) + if dataset_yaml: + self._clear_rgb_cache_for_dataset(dataset_yaml) + def _export_labels_for_split( self, split_name: str, @@ -1165,10 +1171,43 @@ class TrainingTab(QWidget): ) -> Path: """Prepare dataset for training. - Note: With proper 16-bit TIFF support in YOLOWrapper, we no longer need - to create RGB-converted copies of the dataset. Images are handled directly. + For 16-bit TIFF files: creates 3-channel float32 TIFF versions for training. + This is necessary because YOLO's training dataloader loads images directly + from disk and expects 3 channels. """ - return dataset_yaml + dataset_info = dataset_info or ( + self.selected_dataset + if self.selected_dataset + and self.selected_dataset.get("yaml_path") == str(dataset_yaml) + else self._parse_dataset_yaml(dataset_yaml) + ) + + train_split = dataset_info.get("splits", {}).get("train") or {} + images_path_str = train_split.get("path") + if not images_path_str: + return dataset_yaml + + images_path = Path(images_path_str) + if not images_path.exists(): + return dataset_yaml + + # Check if dataset has 16-bit TIFF files that need 3-channel conversion + if not self._dataset_has_16bit_tiff(images_path): + return dataset_yaml + + cache_root = self._get_float32_cache_root(dataset_yaml) + float32_yaml = cache_root / "data.yaml" + if float32_yaml.exists(): + self._append_training_log( + f"Detected 16-bit TIFF dataset; reusing float32 3-channel cache at {cache_root}" + ) + return float32_yaml + + self._append_training_log( + f"Detected 16-bit TIFF dataset; creating float32 3-channel cache at {cache_root}" + ) + self._build_float32_dataset(cache_root, dataset_info) + return float32_yaml def _compose_stage_plan(self, params: Dict[str, Any]) -> List[Dict[str, Any]]: two_stage = params.get("two_stage") or {} @@ -1254,6 +1293,132 @@ class TrainingTab(QWidget): f" • {stage_label}: epochs={epochs}, lr0={lr0}, patience={patience}, freeze={freeze}" ) + def _get_float32_cache_root(self, dataset_yaml: Path) -> Path: + """Get cache directory for float32 3-channel converted datasets.""" + cache_base = Path("data/datasets/_float32_cache") + cache_base.mkdir(parents=True, exist_ok=True) + key = hashlib.md5(str(dataset_yaml.parent.resolve()).encode()).hexdigest()[:8] + return cache_base / f"{dataset_yaml.parent.name}_{key}" + + def _clear_rgb_cache_for_dataset(self, dataset_yaml: Path): + """Clear float32 cache for dataset.""" + cache_root = self._get_float32_cache_root(dataset_yaml) + if cache_root.exists(): + try: + shutil.rmtree(cache_root) + logger.debug(f"Removed float32 cache at {cache_root}") + except OSError as exc: + logger.warning(f"Failed to remove float32 cache {cache_root}: {exc}") + + def _dataset_has_16bit_tiff(self, images_dir: Path) -> bool: + """Check if dataset contains 16-bit TIFF files.""" + sample_image = self._find_first_image(images_dir) + if not sample_image: + return False + try: + if sample_image.suffix.lower() not in [".tif", ".tiff"]: + return False + img = Image(sample_image) + return img.dtype == np.uint16 + except Exception as exc: + logger.warning(f"Failed to inspect image {sample_image}: {exc}") + return False + + def _find_first_image(self, directory: Path) -> Optional[Path]: + """Find first image in directory.""" + if not directory.exists(): + return None + for path in directory.rglob("*"): + if path.is_file() and path.suffix.lower() in self.allowed_extensions: + return path + return None + + def _build_float32_dataset(self, cache_root: Path, dataset_info: Dict[str, Any]): + """Build float32 3-channel version of 16-bit TIFF dataset.""" + if cache_root.exists(): + shutil.rmtree(cache_root) + cache_root.mkdir(parents=True, exist_ok=True) + + splits = dataset_info.get("splits", {}) + for split_name in ("train", "val", "test"): + split_entry = splits.get(split_name) + if not split_entry: + continue + images_src = Path(split_entry.get("path", "")) + if not images_src.exists(): + continue + images_dst = cache_root / split_name / "images" + self._convert_16bit_to_float32_3ch(images_src, images_dst) + + labels_src = self._infer_labels_dir(images_src) + if labels_src.exists(): + labels_dst = cache_root / split_name / "labels" + self._copy_labels(labels_src, labels_dst) + + class_names = dataset_info.get("class_names") or [] + names_map = {idx: name for idx, name in enumerate(class_names)} + num_classes = dataset_info.get("num_classes") or len(class_names) + + yaml_payload: Dict[str, Any] = { + "path": cache_root.as_posix(), + "names": names_map, + "nc": num_classes, + } + + for split_name in ("train", "val", "test"): + images_dir = cache_root / split_name / "images" + if images_dir.exists(): + yaml_payload[split_name] = f"{split_name}/images" + + with open(cache_root / "data.yaml", "w", encoding="utf-8") as handle: + yaml.safe_dump(yaml_payload, handle, sort_keys=False) + + def _convert_16bit_to_float32_3ch(self, src_dir: Path, dst_dir: Path): + """Convert 16-bit TIFF images to float32 [0-1] 3-channel TIFFs. + + This preserves the full dynamic range (no uint8 conversion) while + creating the 3-channel format that YOLO training expects. + """ + for src in src_dir.rglob("*"): + if not src.is_file() or src.suffix.lower() not in self.allowed_extensions: + continue + relative = src.relative_to(src_dir) + dst = dst_dir / relative.with_suffix(".tif") + dst.parent.mkdir(parents=True, exist_ok=True) + try: + img_obj = Image(src) + + # Check if it's a 16-bit TIFF + is_16bit_tiff = ( + src.suffix.lower() in [".tif", ".tiff"] + and img_obj.dtype == np.uint16 + ) + + if is_16bit_tiff: + # Convert to float32 [0-1] + float_data = img_obj.to_normalized_float32() + + # Replicate to 3 channels + if len(float_data.shape) == 2: + # H,W → H,W,3 + float_3ch = np.stack([float_data] * 3, axis=-1) + elif len(float_data.shape) == 3 and float_data.shape[2] == 1: + # H,W,1 → H,W,3 + float_3ch = np.repeat(float_data, 3, axis=2) + else: + # Already multi-channel + float_3ch = float_data + + # Save as float32 TIFF (preserves full precision) + tifffile.imwrite(dst, float_3ch.astype(np.float32)) + logger.debug(f"Converted {src} to float32 3-channel TIFF at {dst}") + else: + # For non-16-bit images, just copy + shutil.copy2(src, dst) + + except Exception as exc: + logger.warning(f"Failed to convert {src}: {exc}") + def _copy_labels(self, labels_src: Path, labels_dst: Path): label_files = list(labels_src.rglob("*.txt")) for label_file in label_files: @@ -1350,6 +1515,10 @@ class TrainingTab(QWidget): self._export_labels_from_database(dataset_info) dataset_to_use = self._prepare_dataset_for_training(dataset_path, dataset_info) + if dataset_to_use != dataset_path: + self._append_training_log( + f"Using float32 3-channel dataset at {dataset_to_use.parent}" + ) params = self._collect_training_params() stage_plan = self._compose_stage_plan(params) diff --git a/tests/test_pyside_freehand_tool b/tests/test_pyside_freehand_tool new file mode 100644 index 0000000..5fd38c4 --- /dev/null +++ b/tests/test_pyside_freehand_tool @@ -0,0 +1,1774 @@ +[ + [ + [ + 374.08203125, + 166.73828125 + ], + [ + 371.890625, + 167.63671875 + ], + [ + 369.890625, + 168.63671875 + ], + [ + 366.890625, + 169.63671875 + ], + [ + 363.890625, + 169.63671875 + ], + [ + 361.890625, + 170.63671875 + ], + [ + 359.890625, + 171.63671875 + ], + [ + 357.890625, + 171.63671875 + ], + [ + 355.890625, + 172.63671875 + ], + [ + 354.10546875, + 173.6171875 + ], + [ + 352.19921875, + 174.5703125 + ], + [ + 349.19921875, + 175.5703125 + ], + [ + 347.19921875, + 176.5703125 + ], + [ + 345.19921875, + 178.5703125 + ], + [ + 343.19921875, + 180.5703125 + ], + [ + 341.2109375, + 182.5546875 + ], + [ + 340.29296875, + 184.39453125 + ], + [ + 340.29296875, + 187.30078125 + ], + [ + 339.29296875, + 189.30078125 + ], + [ + 339.29296875, + 191.30078125 + ], + [ + 339.29296875, + 193.30078125 + ], + [ + 339.29296875, + 195.30078125 + ], + [ + 339.29296875, + 197.30078125 + ], + [ + 340.29296875, + 199.30078125 + ], + [ + 342.2734375, + 201.28125 + ], + [ + 344.2734375, + 203.26171875 + ], + [ + 346.2734375, + 206.26171875 + ], + [ + 348.2734375, + 207.26171875 + ], + [ + 349.2734375, + 209.26171875 + ], + [ + 351.2734375, + 210.26171875 + ], + [ + 353.25, + 211.25 + ], + [ + 355.25, + 212.25 + ], + [ + 357.25, + 213.25 + ], + [ + 359.25, + 214.25 + ], + [ + 361.25, + 215.25 + ], + [ + 363.25, + 216.25 + ], + [ + 365.25, + 216.25 + ], + [ + 367.25, + 217.25 + ], + [ + 369.25, + 217.25 + ], + [ + 371.25, + 217.25 + ], + [ + 373.25, + 217.25 + ], + [ + 376.25, + 217.25 + ], + [ + 379.25, + 218.25 + ], + [ + 382.25, + 218.25 + ], + [ + 384.25, + 218.25 + ], + [ + 386.25, + 217.25 + ], + [ + 389.25, + 217.25 + ], + [ + 391.25, + 217.25 + ], + [ + 393.25, + 216.25 + ], + [ + 395.25, + 216.25 + ], + [ + 397.25, + 216.25 + ], + [ + 400.25, + 215.25 + ], + [ + 403.25, + 215.25 + ], + [ + 405.09375, + 214.3828125 + ], + [ + 406.015625, + 212.4609375 + ], + [ + 408.015625, + 211.4609375 + ], + [ + 410.015625, + 210.4609375 + ], + [ + 410.015625, + 207.65234375 + ], + [ + 410.9375, + 205.84375 + ], + [ + 410.9375, + 202.93359375 + ], + [ + 410.9375, + 200.2578125 + ], + [ + 410.9375, + 197.87109375 + ], + [ + 410.9375, + 195.87109375 + ], + [ + 409.9375, + 193.87109375 + ], + [ + 409.9375, + 191.87109375 + ], + [ + 409.9375, + 189.87109375 + ], + [ + 408.9375, + 187.87109375 + ], + [ + 408.9375, + 185.87109375 + ], + [ + 407.9375, + 183.87109375 + ], + [ + 406.9375, + 181.87109375 + ], + [ + 406.9375, + 179.87109375 + ], + [ + 405.9375, + 177.87109375 + ], + [ + 405.9375, + 175.87109375 + ], + [ + 404.9375, + 173.87109375 + ], + [ + 403.9375, + 170.87109375 + ], + [ + 403.9375, + 168.87109375 + ], + [ + 402.9375, + 166.87109375 + ], + [ + 401.953125, + 163.8984375 + ], + [ + 400.96484375, + 160.92578125 + ], + [ + 400.0390625, + 158.296875 + ], + [ + 398.02734375, + 158.296875 + ], + [ + 395.828125, + 158.296875 + ], + [ + 392.9296875, + 158.296875 + ], + [ + 390.9296875, + 159.296875 + ], + [ + 388.94140625, + 160.296875 + ], + [ + 386.4609375, + 161.95703125 + ], + [ + 384.70703125, + 163.1015625 + ], + [ + 382.28125, + 163.98046875 + ], + [ + 380.28125, + 164.98046875 + ], + [ + 378.28125, + 166.98046875 + ], + [ + 375.66015625, + 167.84765625 + ], + [ + 372.6875, + 167.84765625 + ], + [ + 370.69921875, + 168.84765625 + ], + [ + 374.08203125, + 166.73828125 + ] + ], + [ + [ + 444.55078125, + 253.7109375 + ], + [ + 443.59765625, + 257.03515625 + ], + [ + 441.59765625, + 259.03515625 + ], + [ + 439.59765625, + 260.03515625 + ], + [ + 437.59765625, + 261.03515625 + ], + [ + 436.59765625, + 263.03515625 + ], + [ + 434.59765625, + 264.03515625 + ], + [ + 431.59765625, + 267.03515625 + ], + [ + 428.59765625, + 269.03515625 + ], + [ + 426.59765625, + 271.03515625 + ], + [ + 423.59765625, + 273.03515625 + ], + [ + 422.59765625, + 275.03515625 + ], + [ + 420.59765625, + 278.03515625 + ], + [ + 419.59765625, + 280.03515625 + ], + [ + 418.59765625, + 282.03515625 + ], + [ + 418.59765625, + 284.03515625 + ], + [ + 417.59765625, + 286.03515625 + ], + [ + 416.59765625, + 288.03515625 + ], + [ + 416.59765625, + 290.6640625 + ], + [ + 417.5859375, + 292.64453125 + ], + [ + 419.5859375, + 293.64453125 + ], + [ + 420.5859375, + 295.64453125 + ], + [ + 422.5859375, + 298.64453125 + ], + [ + 424.5859375, + 300.64453125 + ], + [ + 426.5859375, + 301.64453125 + ], + [ + 427.5859375, + 303.64453125 + ], + [ + 429.5859375, + 304.64453125 + ], + [ + 432.5859375, + 305.64453125 + ], + [ + 436.59375, + 306.64453125 + ], + [ + 441.875, + 307.69921875 + ], + [ + 444.9453125, + 308.72265625 + ], + [ + 448.9453125, + 308.72265625 + ], + [ + 452.95703125, + 309.7265625 + ], + [ + 455.9609375, + 309.7265625 + ], + [ + 457.9609375, + 310.7265625 + ], + [ + 459.9609375, + 310.7265625 + ], + [ + 461.9609375, + 310.7265625 + ], + [ + 464.62109375, + 308.73828125 + ], + [ + 465.62109375, + 305.73828125 + ], + [ + 466.62109375, + 303.73828125 + ], + [ + 467.62109375, + 301.73828125 + ], + [ + 468.62109375, + 299.73828125 + ], + [ + 469.62109375, + 296.73828125 + ], + [ + 471.62109375, + 294.73828125 + ], + [ + 472.62109375, + 291.73828125 + ], + [ + 473.62109375, + 288.73828125 + ], + [ + 475.62109375, + 286.73828125 + ], + [ + 476.62109375, + 284.73828125 + ], + [ + 477.62109375, + 281.73828125 + ], + [ + 479.62109375, + 280.73828125 + ], + [ + 481.62109375, + 277.73828125 + ], + [ + 483.62109375, + 275.73828125 + ], + [ + 484.62109375, + 273.73828125 + ], + [ + 484.62109375, + 270.73828125 + ], + [ + 485.62109375, + 268.75 + ], + [ + 485.62109375, + 265.9140625 + ], + [ + 485.62109375, + 263.9140625 + ], + [ + 484.62109375, + 261.9140625 + ], + [ + 483.62109375, + 259.9140625 + ], + [ + 482.62109375, + 257.9140625 + ], + [ + 482.62109375, + 255.9140625 + ], + [ + 480.69140625, + 253.98046875 + ], + [ + 478.734375, + 253.0234375 + ], + [ + 476.734375, + 251.0234375 + ], + [ + 474.74609375, + 250.0390625 + ], + [ + 472.74609375, + 250.0390625 + ], + [ + 469.74609375, + 250.0390625 + ], + [ + 467.74609375, + 250.0390625 + ], + [ + 464.74609375, + 250.0390625 + ], + [ + 461.74609375, + 250.0390625 + ], + [ + 459.74609375, + 250.0390625 + ], + [ + 457.74609375, + 251.0390625 + ], + [ + 455.74609375, + 251.0390625 + ], + [ + 453.74609375, + 251.0390625 + ], + [ + 452.3359375, + 252.4765625 + ], + [ + 449.39453125, + 253.4765625 + ], + [ + 447.39453125, + 254.4765625 + ], + [ + 445.39453125, + 254.4765625 + ], + [ + 443.40625, + 255.4765625 + ], + [ + 444.55078125, + 253.7109375 + ] + ], + [ + [ + 278.51171875, + 276.58984375 + ], + [ + 278.51171875, + 278.94140625 + ], + [ + 278.51171875, + 280.94140625 + ], + [ + 279.51171875, + 283.94140625 + ], + [ + 279.51171875, + 285.94140625 + ], + [ + 279.51171875, + 287.94140625 + ], + [ + 279.51171875, + 290.94140625 + ], + [ + 280.51171875, + 292.94140625 + ], + [ + 280.51171875, + 294.94140625 + ], + [ + 280.51171875, + 296.94140625 + ], + [ + 281.515625, + 300.94921875 + ], + [ + 282.515625, + 302.94921875 + ], + [ + 283.515625, + 305.94921875 + ], + [ + 285.515625, + 308.94921875 + ], + [ + 287.515625, + 311.94921875 + ], + [ + 289.515625, + 312.94921875 + ], + [ + 291.515625, + 315.94921875 + ], + [ + 293.515625, + 316.94921875 + ], + [ + 295.515625, + 318.94921875 + ], + [ + 297.515625, + 319.94921875 + ], + [ + 300.515625, + 321.94921875 + ], + [ + 304.5546875, + 323.96875 + ], + [ + 309.98828125, + 326.14453125 + ], + [ + 313.08984375, + 327.17578125 + ], + [ + 317.08984375, + 328.17578125 + ], + [ + 320.08984375, + 329.17578125 + ], + [ + 323.08984375, + 330.17578125 + ], + [ + 326.08984375, + 331.17578125 + ], + [ + 329.08984375, + 331.17578125 + ], + [ + 331.08984375, + 331.17578125 + ], + [ + 333.08984375, + 332.17578125 + ], + [ + 335.08984375, + 332.17578125 + ], + [ + 337.08984375, + 332.17578125 + ], + [ + 339.08984375, + 331.17578125 + ], + [ + 340.08984375, + 329.17578125 + ], + [ + 342.08984375, + 327.17578125 + ], + [ + 344.08984375, + 326.17578125 + ], + [ + 346.08984375, + 323.17578125 + ], + [ + 348.08984375, + 321.17578125 + ], + [ + 349.08984375, + 318.17578125 + ], + [ + 350.08984375, + 316.17578125 + ], + [ + 351.08984375, + 314.17578125 + ], + [ + 351.08984375, + 311.203125 + ], + [ + 351.08984375, + 309.203125 + ], + [ + 350.08984375, + 307.203125 + ], + [ + 350.08984375, + 305.203125 + ], + [ + 350.08984375, + 303.203125 + ], + [ + 349.08984375, + 301.203125 + ], + [ + 349.08984375, + 299.203125 + ], + [ + 348.08984375, + 297.203125 + ], + [ + 346.08984375, + 296.203125 + ], + [ + 344.08984375, + 294.203125 + ], + [ + 343.08984375, + 292.21484375 + ], + [ + 341.1015625, + 290.2265625 + ], + [ + 339.1015625, + 288.2265625 + ], + [ + 336.1015625, + 286.2265625 + ], + [ + 333.1015625, + 284.2265625 + ], + [ + 331.1015625, + 283.2265625 + ], + [ + 329.1015625, + 282.2265625 + ], + [ + 328.1015625, + 280.2265625 + ], + [ + 326.11328125, + 279.23828125 + ], + [ + 323.11328125, + 277.23828125 + ], + [ + 321.11328125, + 275.23828125 + ], + [ + 319.11328125, + 273.23828125 + ], + [ + 317.11328125, + 271.23828125 + ], + [ + 315.11328125, + 270.23828125 + ], + [ + 313.11328125, + 268.23828125 + ], + [ + 311.11328125, + 267.23828125 + ], + [ + 309.11328125, + 266.23828125 + ], + [ + 307.11328125, + 266.23828125 + ], + [ + 305.11328125, + 265.23828125 + ], + [ + 303.11328125, + 264.23828125 + ], + [ + 301.23046875, + 263.296875 + ], + [ + 299.23046875, + 263.296875 + ], + [ + 297.06640625, + 263.296875 + ], + [ + 295.3359375, + 265.03125 + ], + [ + 293.3359375, + 267.03125 + ], + [ + 291.3359375, + 268.03125 + ], + [ + 289.3359375, + 270.03125 + ], + [ + 287.3359375, + 271.03125 + ], + [ + 285.52734375, + 271.8984375 + ], + [ + 283.67578125, + 272.765625 + ], + [ + 282.6875, + 274.6796875 + ], + [ + 280.16015625, + 275.66796875 + ], + [ + 278.51171875, + 276.58984375 + ] + ], + [ + [ + 552.5, + 260.89453125 + ], + [ + 550.59765625, + 264.21484375 + ], + [ + 548.59765625, + 267.21484375 + ], + [ + 547.59765625, + 269.21484375 + ], + [ + 546.59765625, + 271.21484375 + ], + [ + 544.59765625, + 273.21484375 + ], + [ + 543.59765625, + 276.21484375 + ], + [ + 542.59765625, + 278.21484375 + ], + [ + 541.59765625, + 280.21484375 + ], + [ + 540.59765625, + 282.21484375 + ], + [ + 540.59765625, + 284.21484375 + ], + [ + 540.59765625, + 287.11328125 + ], + [ + 541.59765625, + 290.08984375 + ], + [ + 543.59765625, + 292.08984375 + ], + [ + 545.59765625, + 293.08984375 + ], + [ + 547.59765625, + 294.08984375 + ], + [ + 550.59765625, + 296.08984375 + ], + [ + 552.59765625, + 297.08984375 + ], + [ + 556.59765625, + 299.08984375 + ], + [ + 558.59765625, + 300.08984375 + ], + [ + 561.59765625, + 301.08984375 + ], + [ + 563.59765625, + 302.08984375 + ], + [ + 566.59765625, + 303.08984375 + ], + [ + 568.59765625, + 303.08984375 + ], + [ + 570.59765625, + 303.08984375 + ], + [ + 572.59765625, + 303.08984375 + ], + [ + 575.375, + 302.30078125 + ], + [ + 576.36328125, + 299.53515625 + ], + [ + 576.36328125, + 297.53515625 + ], + [ + 577.36328125, + 295.53515625 + ], + [ + 577.36328125, + 293.53515625 + ], + [ + 577.36328125, + 289.5625 + ], + [ + 577.36328125, + 286.5625 + ], + [ + 577.36328125, + 283.5625 + ], + [ + 578.36328125, + 281.5625 + ], + [ + 578.36328125, + 279.5625 + ], + [ + 578.36328125, + 277.5625 + ], + [ + 578.36328125, + 274.5625 + ], + [ + 578.36328125, + 271.94140625 + ], + [ + 577.36328125, + 269.94140625 + ], + [ + 576.36328125, + 267.94140625 + ], + [ + 574.36328125, + 265.94140625 + ], + [ + 573.36328125, + 263.94140625 + ], + [ + 571.36328125, + 262.94140625 + ], + [ + 569.36328125, + 260.94140625 + ], + [ + 567.375, + 259.94140625 + ], + [ + 566.375, + 257.953125 + ], + [ + 564.375, + 257.953125 + ], + [ + 561.78125, + 258.71484375 + ], + [ + 559.890625, + 259.6953125 + ], + [ + 557.44921875, + 260.44140625 + ], + [ + 555.55078125, + 261.43359375 + ], + [ + 553.53125, + 261.43359375 + ], + [ + 552.5, + 260.89453125 + ] + ], + [ + [ + 546.4140625, + 148.60546875 + ], + [ + 544.125, + 149.9921875 + ], + [ + 542.125, + 152.9921875 + ], + [ + 539.125, + 155.9921875 + ], + [ + 538.125, + 157.9921875 + ], + [ + 536.125, + 159.9921875 + ], + [ + 534.125, + 161.9921875 + ], + [ + 532.125, + 163.9921875 + ], + [ + 530.125, + 165.9921875 + ], + [ + 528.125, + 167.9921875 + ], + [ + 528.125, + 170.96484375 + ], + [ + 528.125, + 173.64453125 + ], + [ + 531.0625, + 177.58203125 + ], + [ + 534.0703125, + 180.59375 + ], + [ + 536.078125, + 182.59765625 + ], + [ + 540.140625, + 185.64453125 + ], + [ + 543.3125, + 188.81640625 + ], + [ + 546.32421875, + 190.82421875 + ], + [ + 548.32421875, + 191.82421875 + ], + [ + 550.32421875, + 192.82421875 + ], + [ + 552.296875, + 193.8125 + ], + [ + 555.10546875, + 193.8125 + ], + [ + 557.91796875, + 192.8125 + ], + [ + 559.91796875, + 191.8125 + ], + [ + 560.90234375, + 189.82421875 + ], + [ + 561.90234375, + 187.82421875 + ], + [ + 562.90234375, + 185.82421875 + ], + [ + 563.90234375, + 183.82421875 + ], + [ + 564.90234375, + 181.82421875 + ], + [ + 565.90234375, + 179.82421875 + ], + [ + 566.90234375, + 177.82421875 + ], + [ + 567.90234375, + 175.82421875 + ], + [ + 568.90234375, + 173.82421875 + ], + [ + 569.90234375, + 171.82421875 + ], + [ + 570.90234375, + 169.82421875 + ], + [ + 570.90234375, + 167.82421875 + ], + [ + 571.90234375, + 165.82421875 + ], + [ + 571.90234375, + 162.86328125 + ], + [ + 570.1171875, + 160.2578125 + ], + [ + 567.1171875, + 157.2578125 + ], + [ + 565.1171875, + 156.2578125 + ], + [ + 563.1171875, + 155.2578125 + ], + [ + 561.1171875, + 153.2578125 + ], + [ + 559.1171875, + 152.2578125 + ], + [ + 557.3046875, + 151.2578125 + ], + [ + 555.03125, + 149.8671875 + ], + [ + 552.66015625, + 148.234375 + ], + [ + 553.2109375, + 150.23046875 + ], + [ + 546.4140625, + 148.60546875 + ] + ], + [ + [ + 462.41015625, + 129.0859375 + ], + [ + 461.41015625, + 132.3984375 + ], + [ + 461.41015625, + 134.3984375 + ], + [ + 461.41015625, + 136.3984375 + ], + [ + 460.41015625, + 138.3984375 + ], + [ + 460.41015625, + 141.3984375 + ], + [ + 460.41015625, + 144.3984375 + ], + [ + 460.41015625, + 146.3984375 + ], + [ + 460.41015625, + 148.3984375 + ], + [ + 460.41015625, + 150.3984375 + ], + [ + 460.41015625, + 152.3984375 + ], + [ + 460.41015625, + 155.3984375 + ], + [ + 461.41015625, + 157.3984375 + ], + [ + 461.41015625, + 159.3984375 + ], + [ + 461.41015625, + 161.3984375 + ], + [ + 462.41015625, + 163.3984375 + ], + [ + 462.41015625, + 165.3984375 + ], + [ + 463.41015625, + 167.3984375 + ], + [ + 464.41015625, + 170.3984375 + ], + [ + 465.41015625, + 172.3984375 + ], + [ + 465.41015625, + 174.3984375 + ], + [ + 466.41015625, + 176.3984375 + ], + [ + 467.3984375, + 178.3828125 + ], + [ + 469.30078125, + 179.3359375 + ], + [ + 471.30078125, + 179.3359375 + ], + [ + 473.30078125, + 180.3359375 + ], + [ + 476.30078125, + 180.3359375 + ], + [ + 478.30078125, + 180.3359375 + ], + [ + 480.30078125, + 180.3359375 + ], + [ + 483.30078125, + 180.3359375 + ], + [ + 485.30078125, + 180.3359375 + ], + [ + 487.30078125, + 179.3359375 + ], + [ + 488.30078125, + 177.3359375 + ], + [ + 490.30078125, + 176.3359375 + ], + [ + 492.30078125, + 175.3359375 + ], + [ + 493.30078125, + 173.3359375 + ], + [ + 493.30078125, + 171.3359375 + ], + [ + 493.30078125, + 168.3359375 + ], + [ + 494.30078125, + 166.3359375 + ], + [ + 494.30078125, + 164.3359375 + ], + [ + 494.30078125, + 161.46484375 + ], + [ + 494.30078125, + 158.46484375 + ], + [ + 494.30078125, + 156.46484375 + ], + [ + 493.30078125, + 153.46484375 + ], + [ + 493.30078125, + 150.46484375 + ], + [ + 493.30078125, + 148.46484375 + ], + [ + 492.30078125, + 145.46484375 + ], + [ + 492.30078125, + 142.46484375 + ], + [ + 492.30078125, + 140.46484375 + ], + [ + 491.30078125, + 138.46484375 + ], + [ + 487.921875, + 139.1171875 + ], + [ + 485.921875, + 140.1171875 + ], + [ + 482.921875, + 141.1171875 + ], + [ + 480.921875, + 142.1171875 + ], + [ + 478.921875, + 143.1171875 + ], + [ + 478.921875, + 145.35546875 + ], + [ + 477.9765625, + 147.94140625 + ], + [ + 476.98828125, + 149.87890625 + ], + [ + 475.02734375, + 149.359375 + ], + [ + 474.07421875, + 146.86328125 + ], + [ + 473.15234375, + 144.953125 + ], + [ + 473.15234375, + 141.98828125 + ], + [ + 473.15234375, + 139.015625 + ], + [ + 473.15234375, + 137.015625 + ], + [ + 473.15234375, + 134.0390625 + ], + [ + 473.15234375, + 131.078125 + ], + [ + 474.15234375, + 129.08984375 + ], + [ + 474.15234375, + 127.08984375 + ], + [ + 474.15234375, + 124.5859375 + ], + [ + 471.8515625, + 123.68359375 + ], + [ + 469.8515625, + 122.68359375 + ], + [ + 467.87890625, + 121.6953125 + ], + [ + 465.87890625, + 120.6953125 + ], + [ + 463.87890625, + 120.6953125 + ], + [ + 462.92578125, + 122.76171875 + ], + [ + 462.92578125, + 124.86328125 + ], + [ + 462.92578125, + 127.3046875 + ], + [ + 462.41015625, + 129.0859375 + ] + ] +] \ No newline at end of file diff --git a/tests/test_training_dataset_prep.py b/tests/test_training_dataset_prep.py new file mode 100644 index 0000000..74465d4 --- /dev/null +++ b/tests/test_training_dataset_prep.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +""" +Test script for training dataset preparation with 16-bit TIFFs. +""" + +import numpy as np +import tifffile +from pathlib import Path +import tempfile +import sys +import os +import shutil + +# Add parent directory to path to import modules +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from src.utils.image import Image + + +def test_float32_3ch_conversion(): + """Test conversion of 16-bit TIFF to float32 3-channel TIFF.""" + print("\n=== Testing Float32 3-Channel Conversion ===") + + # Create temporary directory structure + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = Path(tmpdir) + src_dir = tmpdir / "original" + dst_dir = tmpdir / "converted" + src_dir.mkdir() + dst_dir.mkdir() + + # Create test 16-bit TIFF + test_data = np.zeros((100, 100), dtype=np.uint16) + for i in range(100): + for j in range(100): + test_data[i, j] = int((i + j) / 198 * 65535) + + test_file = src_dir / "test_16bit.tif" + tifffile.imwrite(test_file, test_data) + print(f"Created test 16-bit TIFF: {test_file}") + print(f" Shape: {test_data.shape}") + print(f" Dtype: {test_data.dtype}") + print(f" Range: [{test_data.min()}, {test_data.max()}]") + + # Simulate the conversion process + print("\nConverting to float32 3-channel...") + img_obj = Image(test_file) + + # Convert to float32 [0-1] + float_data = img_obj.to_normalized_float32() + + # Replicate to 3 channels + if len(float_data.shape) == 2: + float_3ch = np.stack([float_data] * 3, axis=-1) + else: + float_3ch = float_data + + # Save as float32 TIFF + output_file = dst_dir / "test_float32_3ch.tif" + tifffile.imwrite(output_file, float_3ch.astype(np.float32)) + print(f"Saved float32 3-channel TIFF: {output_file}") + + # Verify the output + loaded = tifffile.imread(output_file) + print(f"\nVerifying output:") + print(f" Shape: {loaded.shape}") + print(f" Dtype: {loaded.dtype}") + print(f" Channels: {loaded.shape[2] if len(loaded.shape) == 3 else 1}") + print(f" Range: [{loaded.min():.6f}, {loaded.max():.6f}]") + print(f" Unique values: {len(np.unique(loaded[:,:,0]))}") + + # Assertions + assert loaded.dtype == np.float32, f"Expected float32, got {loaded.dtype}" + assert loaded.shape[2] == 3, f"Expected 3 channels, got {loaded.shape[2]}" + assert ( + 0.0 <= loaded.min() <= loaded.max() <= 1.0 + ), f"Expected [0,1] range, got [{loaded.min()}, {loaded.max()}]" + + # Verify all channels are identical (replicated grayscale) + assert np.array_equal( + loaded[:, :, 0], loaded[:, :, 1] + ), "Channel 0 and 1 should be identical" + assert np.array_equal( + loaded[:, :, 0], loaded[:, :, 2] + ), "Channel 0 and 2 should be identical" + + # Verify float32 precision (not quantized to uint8 steps) + unique_vals = len(np.unique(loaded[:, :, 0])) + print(f"\n Precision check:") + print(f" Unique values in channel: {unique_vals}") + print(f" Source unique values: {len(np.unique(test_data))}") + + # The final unique values should match source (no loss from conversion) + assert unique_vals == len( + np.unique(test_data) + ), f"Expected {len(np.unique(test_data))} unique values, got {unique_vals}" + + print("\n✓ All conversion tests passed!") + print(" - Float32 dtype preserved") + print(" - 3 channels created") + print(" - Range [0-1] maintained") + print(" - No precision loss from conversion") + print(" - Channels properly replicated") + + return True + + +if __name__ == "__main__": + try: + success = test_float32_3ch_conversion() + sys.exit(0 if success else 1) + except Exception as e: + print(f"\n✗ Test failed with error: {e}") + import traceback + + traceback.print_exc() + sys.exit(1)