Multi-view reconstruction and Meshing

0. Import

%matplotlib inline

import openalea.phenomenal.data as phm_data
import openalea.phenomenal.display as phm_display
import openalea.phenomenal.object as phm_obj
import openalea.phenomenal.multi_view_reconstruction as phm_mvr
import openalea.phenomenal.mesh as phm_mesh
import openalea.phenomenal.display.notebook as phm_display_notebook
from openalea.phenotyping_data.fetch import fetch_all_data

1. Prerequisites

1.1 Load data

plant_number = 2  # Available : 1, 2, 3, 4 or 5
data_dir = fetch_all_data(f"plant_{plant_number}")
bin_image_paths = phm_data.bin_image_paths(data_dir)
calibration = phm_data.load_calibration(data_dir)
bin_image_paths,calibration

Downloading file 'plant_2/bin/side/0.png' from 'https://raw.githubusercontent.com/openalea/phenotyping_data/main/data/plant_2/bin/side/0.png' to '/home/docs/.cache/phenotyping_data'.
Downloading file 'plant_2/bin/side/120.png' from 'https://raw.githubusercontent.com/openalea/phenotyping_data/main/data/plant_2/bin/side/120.png' to '/home/docs/.cache/phenotyping_data'.
Downloading file 'plant_2/bin/side/150.png' from 'https://raw.githubusercontent.com/openalea/phenotyping_data/main/data/plant_2/bin/side/150.png' to '/home/docs/.cache/phenotyping_data'.
Downloading file 'plant_2/bin/side/180.png' from 'https://raw.githubusercontent.com/openalea/phenotyping_data/main/data/plant_2/bin/side/180.png' to '/home/docs/.cache/phenotyping_data'.
Downloading file 'plant_2/bin/side/210.png' from 'https://raw.githubusercontent.com/openalea/phenotyping_data/main/data/plant_2/bin/side/210.png' to '/home/docs/.cache/phenotyping_data'.
Downloading file 'plant_2/bin/side/240.png' from 'https://raw.githubusercontent.com/openalea/phenotyping_data/main/data/plant_2/bin/side/240.png' to '/home/docs/.cache/phenotyping_data'.
Downloading file 'plant_2/bin/side/270.png' from 'https://raw.githubusercontent.com/openalea/phenotyping_data/main/data/plant_2/bin/side/270.png' to '/home/docs/.cache/phenotyping_data'.
Downloading file 'plant_2/bin/side/30.png' from 'https://raw.githubusercontent.com/openalea/phenotyping_data/main/data/plant_2/bin/side/30.png' to '/home/docs/.cache/phenotyping_data'.
Downloading file 'plant_2/bin/side/300.png' from 'https://raw.githubusercontent.com/openalea/phenotyping_data/main/data/plant_2/bin/side/300.png' to '/home/docs/.cache/phenotyping_data'.
Downloading file 'plant_2/bin/side/330.png' from 'https://raw.githubusercontent.com/openalea/phenotyping_data/main/data/plant_2/bin/side/330.png' to '/home/docs/.cache/phenotyping_data'.
Downloading file 'plant_2/bin/side/60.png' from 'https://raw.githubusercontent.com/openalea/phenotyping_data/main/data/plant_2/bin/side/60.png' to '/home/docs/.cache/phenotyping_data'.
Downloading file 'plant_2/bin/side/90.png' from 'https://raw.githubusercontent.com/openalea/phenotyping_data/main/data/plant_2/bin/side/90.png' to '/home/docs/.cache/phenotyping_data'.
Downloading file 'plant_2/bin/top/0.png' from 'https://raw.githubusercontent.com/openalea/phenotyping_data/main/data/plant_2/bin/top/0.png' to '/home/docs/.cache/phenotyping_data'.
Downloading file 'plant_2/calibration/calibration_camera_side.json' from 'https://raw.githubusercontent.com/openalea/phenotyping_data/main/data/plant_2/calibration/calibration_camera_side.json' to '/home/docs/.cache/phenotyping_data'.
Downloading file 'plant_2/calibration/calibration_camera_top.json' from 'https://raw.githubusercontent.com/openalea/phenotyping_data/main/data/plant_2/calibration/calibration_camera_top.json' to '/home/docs/.cache/phenotyping_data'.
Downloading file 'plant_2/voxels/16.npz' from 'https://raw.githubusercontent.com/openalea/phenotyping_data/main/data/plant_2/voxels/16.npz' to '/home/docs/.cache/phenotyping_data'.
Downloading file 'plant_2/voxels/2.npz' from 'https://raw.githubusercontent.com/openalea/phenotyping_data/main/data/plant_2/voxels/2.npz' to '/home/docs/.cache/phenotyping_data'.
Downloading file 'plant_2/voxels/4.npz' from 'https://raw.githubusercontent.com/openalea/phenotyping_data/main/data/plant_2/voxels/4.npz' to '/home/docs/.cache/phenotyping_data'.
Downloading file 'plant_2/voxels/8.npz' from 'https://raw.githubusercontent.com/openalea/phenotyping_data/main/data/plant_2/voxels/8.npz' to '/home/docs/.cache/phenotyping_data'.

(defaultdict(dict,
             {'top': {0: PosixPath('/home/docs/.cache/phenotyping_data/plant_2/bin/top/0.png')},
              'side': {330: PosixPath('/home/docs/.cache/phenotyping_data/plant_2/bin/side/330.png'),
               210: PosixPath('/home/docs/.cache/phenotyping_data/plant_2/bin/side/210.png'),
               30: PosixPath('/home/docs/.cache/phenotyping_data/plant_2/bin/side/30.png'),
               0: PosixPath('/home/docs/.cache/phenotyping_data/plant_2/bin/side/0.png'),
               60: PosixPath('/home/docs/.cache/phenotyping_data/plant_2/bin/side/60.png'),
               270: PosixPath('/home/docs/.cache/phenotyping_data/plant_2/bin/side/270.png'),
               300: PosixPath('/home/docs/.cache/phenotyping_data/plant_2/bin/side/300.png'),
               180: PosixPath('/home/docs/.cache/phenotyping_data/plant_2/bin/side/180.png'),
               90: PosixPath('/home/docs/.cache/phenotyping_data/plant_2/bin/side/90.png'),
               120: PosixPath('/home/docs/.cache/phenotyping_data/plant_2/bin/side/120.png'),
               240: PosixPath('/home/docs/.cache/phenotyping_data/plant_2/bin/side/240.png'),
               150: PosixPath('/home/docs/.cache/phenotyping_data/plant_2/bin/side/150.png')}}),
 <openalea.phenomenal.calibration.object.OldCalibration at 0x7905e404eba0>)

imread = lambda x: phm_data.read_image(x,'L')
sample = {f'{cam}_{ang}': im for cam, ang, im in phm_obj.iter_image_paths(bin_image_paths, imread, angles={0,90})}
phm_display.show_images(sample.values(), list(sample.keys()))

2. Multi-view reconstruction

2.1 Associate images and projection function

image_views = phm_obj.as_image_views(phm_obj.iter_image_paths(bin_image_paths, imread), calibration)
image_views

{'top_0': <openalea.phenomenal.object.imageView.ImageView at 0x7905e3f33cb0>,
 'side_330': <openalea.phenomenal.object.imageView.ImageView at 0x7905e3f67750>,
 'side_210': <openalea.phenomenal.object.imageView.ImageView at 0x7905e3f679d0>,
 'side_30': <openalea.phenomenal.object.imageView.ImageView at 0x7905e3f45940>,
 'side_0': <openalea.phenomenal.object.imageView.ImageView at 0x7905e3f45f30>,
 'side_60': <openalea.phenomenal.object.imageView.ImageView at 0x7905e3e35010>,
 'side_270': <openalea.phenomenal.object.imageView.ImageView at 0x7905e40a1f20>,
 'side_300': <openalea.phenomenal.object.imageView.ImageView at 0x7905e40a3240>,
 'side_180': <openalea.phenomenal.object.imageView.ImageView at 0x7905e3e3ba50>,
 'side_90': <openalea.phenomenal.object.imageView.ImageView at 0x7905e3e3bc50>,
 'side_120': <openalea.phenomenal.object.imageView.ImageView at 0x7905e40dbb60>,
 'side_240': <openalea.phenomenal.object.imageView.ImageView at 0x7905e3e49400>,
 'side_150': <openalea.phenomenal.object.imageView.ImageView at 0x7905e3fd58d0>}

2.2 Do multi-view reconstruction

voxels_size = 16  # mm
error_tolerance = 0
voxel_grid = phm_mvr.reconstruction_3d(
    image_views, voxels_size=voxels_size, error_tolerance=error_tolerance, clear_outside='side')
phm_display_notebook.show_voxel_grid(voxel_grid)

voxel_grid.bounding_box()

((np.float64(-616.0), np.float64(-424.0), np.float64(-696.0)),
 (np.float64(584.0), np.float64(440.0), np.float64(1256.0)))

2.3 Evaluate reprojection

phm_mvr.reconstruction_metrics(voxel_grid, image_views)

{'TP': np.float64(248008.23076923078),
 'FP': np.float64(193156.23076923078),
 'FN': np.float64(9224.0),
 'TN': np.float64(4563115.538461538),
 'precision': np.float64(0.5475525870780391),
 'recall': np.float64(0.958123545199421),
 'IoU': np.float64(0.5349948282149216),
 'Dice': np.float64(0.6957046041484791)}

2.4 Save / Load voxel grid

voxel_grid.write(f"plant_{plant_number}_size_{voxels_size}.npz")

voxel_grid = phm_obj.VoxelGrid.read(f"plant_{plant_number}_size_{voxels_size}.npz")

3.Meshing

vertices, faces = phm_mesh.meshing(
    voxel_grid.to_image_3d(), reduction=0.90, smoothing_iteration=5)
phm_display_notebook.show_mesh(vertices, faces)