Coverage for src/rok4/Pyramid.py: 78%

1"""Provide classes to use pyramid's data.

3The module contains the following classes:

5- `Pyramid` - Data container

6- `Level` - Level of a pyramid

7"""

9from typing import Dict, List, Tuple, Union, Iterator

10import json

11from json.decoder import JSONDecodeError

12import os

13import re

14import numpy

15import zlib

16import io

17import mapbox_vector_tile

18from PIL import Image

20from rok4.Exceptions import *

21from rok4.TileMatrixSet import TileMatrixSet, TileMatrix

22from rok4.Storage import *

23from rok4.Utils import *

26class PyramidType(Enum):

27 """Pyramid's data type"""

29 RASTER = "RASTER"

30 VECTOR = "VECTOR"

33class SlabType(Enum):

34 """Slab's type"""

36 DATA = "DATA" # Slab of data, raster or vector

37 MASK = "MASK" # Slab of mask, only for raster pyramid, image with one band : 0 is nodata, other values are data

40ROK4_IMAGE_HEADER_SIZE = 2048

41"""Slab's header size, 2048 bytes"""

44def b36_number_encode(number: int) -> str:

45 """Convert base-10 number to base-36

47 Used alphabet is '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ'

49 Args:

50 number (int): base-10 number

52 Returns:

53 str: base-36 number

54 """

56 alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"

58 base36 = ""

60 if 0 <= number < len(alphabet):

61 return alphabet[number]

63 while number != 0:

64 number, i = divmod(number, len(alphabet))

65 base36 = alphabet[i] + base36

67 return base36

70def b36_number_decode(number: str) -> int:

71 """Convert base-36 number to base-10

73 Args:

74 number (str): base-36 number

76 Returns:

77 int: base-10 number

78 """

79 return int(number, 36)

82def b36_path_decode(path: str) -> Tuple[int, int]:

83 """Get slab's column and row from a base-36 based path

85 Args:

86 path (str): slab's path

88 Returns:

89 Tuple[int, int]: slab's column and row

90 """

92 path = path.replace("/", "")

93 path = re.sub(r"(\.TIFF?)", "", path.upper())

95 b36_column = ""

96 b36_row = ""

98 while len(path) > 0:

99 b36_column += path[0]

100 b36_row += path[1]

101 path = path[2:]

102

103 return b36_number_decode(b36_column), b36_number_decode(b36_row)

104

105

106def b36_path_encode(column: int, row: int, slashs: int) -> str:

107 """Convert slab indices to base-36 based path, with .tif extension

108

109 Args:

110 column (int): slab's column

111 row (int): slab's row

112 slashs (int): slashs' number (to split path)

113

114 Returns:

115 str: base-36 based path

116 """

117

118 b36_column = b36_number_encode(column)

119 b36_row = b36_number_encode(row)

120

121 max_len = max(slashs + 1, len(b36_column), len(b36_row))

122

123 b36_column = b36_column.rjust(max_len, "0")

124 b36_row = b36_row.rjust(max_len, "0")

125

126 b36_path = ""

127

128 while len(b36_column) > 0:

129 b36_path = b36_row[-1] + b36_path

130 b36_path = b36_column[-1] + b36_path

131

132 b36_column = b36_column[:-1]

133 b36_row = b36_row[:-1]

134

135 if slashs > 0:

136 b36_path = "/" + b36_path

137 slashs -= 1

138

139 return f"{b36_path}.tif"

140

141

142class Level:

143 """A pyramid's level, raster or vector

144

145 Attributes:

146 __id (str): level's identifier. have to exist in the pyramid's used TMS

147 __tile_limits (Dict[str, int]): minimum and maximum tiles' columns and rows of pyramid's content

148 __slab_size (Tuple[int, int]): number of tile in a slab, widthwise and heightwise

149 __tables (List[Dict]): for a VECTOR pyramid, description of vector content, tables and attributes

150 """

151

152 @classmethod

153 def from_descriptor(cls, data: Dict, pyramid: "Pyramid") -> "Level":

154 """Create a pyramid's level from the pyramid's descriptor levels element

155

156 Args:

157 data (Dict): level's information from the pyramid's descriptor

158 pyramid (Pyramid): pyramid containing the level to create

159

160 Raises:

161 Exception: different storage or masks presence between the level and the pyramid

162 MissingAttributeError: Attribute is missing in the content

163

164 Returns:

165 Pyramid: a Level instance

166 """

167 level = cls()

168

169 level.__pyramid = pyramid

170

171 # Attributs communs

172 try:

173 level.__id = data["id"]

174 level.__tile_limits = data["tile_limits"]

175 level.__slab_size = (

176 data["tiles_per_width"],

177 data["tiles_per_height"],

178 )

179

180 # Informations sur le stockage : on les valide et stocke dans la pyramide

181 if pyramid.storage_type.name != data["storage"]["type"]:

182 raise Exception(

183 f"Pyramid {pyramid.descriptor} owns levels using different storage types ({ data['storage']['type'] }) than its one ({pyramid.storage_type.name})"

184 )

185

186 if pyramid.storage_type == StorageType.FILE:

187 pyramid.storage_depth = data["storage"]["path_depth"]

188

189 if "mask_directory" in data["storage"] or "mask_prefix" in data["storage"]:

190 if not pyramid.own_masks:

191 raise Exception(

192 f"Pyramid {pyramid.__descriptor} does not define a mask format but level {level.__id} define mask storage informations"

193 )

194 else:

195 if pyramid.own_masks:

196 raise Exception(

197 f"Pyramid {pyramid.__descriptor} define a mask format but level {level.__id} does not define mask storage informations"

198 )

199

200 except KeyError as e:

201 raise MissingAttributeError(pyramid.descriptor, f"levels[].{e}")

202

203 # Attributs dans le cas d'un niveau vecteur

204 if level.__pyramid.type == PyramidType.VECTOR:

205 try:

206 level.__tables = data["tables"]

207

208 except KeyError as e:

209 raise MissingAttributeError(pyramid.descriptor, f"levels[].{e}")

210

211 return level

212

213 @classmethod

214 def from_other(cls, other: "Level", pyramid: "Pyramid") -> "Level":

215 """Create a pyramid's level from another one

216

217 Args:

218 other (Level): level to clone

219 pyramid (Pyramid): new pyramid containing the new level

220

221 Raises:

222 Exception: different storage or masks presence between the level and the pyramid

223 MissingAttributeError: Attribute is missing in the content

224

225 Returns:

226 Pyramid: a Level instance

227 """

228

229 level = cls()

230

231 # Attributs communs

232 level.__id = other.__id

233 level.__pyramid = pyramid

234 level.__tile_limits = other.__tile_limits

235 level.__slab_size = other.__slab_size

236

237 # Attributs dans le cas d'un niveau vecteur

238 if level.__pyramid.type == PyramidType.VECTOR:

239 level.__tables = other.__tables

240

241 return level

242

243 def __str__(self) -> str:

244 return f"{self.__pyramid.type.name} pyramid's level '{self.__id}' ({self.__pyramid.storage_type.name} storage)"

245

246 @property

247 def serializable(self) -> Dict:

248 """Get the dict version of the pyramid object, pyramid's descriptor compliant

249

250 Returns:

251 Dict: pyramid's descriptor structured object description

252 """

253 serialization = {

254 "id": self.__id,

255 "tiles_per_width": self.__slab_size[0],

256 "tiles_per_height": self.__slab_size[1],

257 "tile_limits": self.__tile_limits,

258 }

259

260 if self.__pyramid.type == PyramidType.VECTOR:

261 serialization["tables"] = self.__tables

262

263 if self.__pyramid.storage_type == StorageType.FILE:

264 serialization["storage"] = {

265 "type": "FILE",

266 "image_directory": f"{self.__pyramid.name}/DATA/{self.__id}",

267 "path_depth": self.__pyramid.storage_depth,

268 }

269 if self.__pyramid.own_masks:

270 serialization["storage"][

271 "mask_directory"

272 ] = f"{self.__pyramid.name}/MASK/{self.__id}"

273

274 elif self.__pyramid.storage_type == StorageType.CEPH:

275 serialization["storage"] = {

276 "type": "CEPH",

277 "image_prefix": f"{self.__pyramid.name}/DATA_{self.__id}",

278 "pool_name": self.__pyramid.storage_root,

279 }

280 if self.__pyramid.own_masks:

281 serialization["storage"]["mask_prefix"] = f"{self.__pyramid.name}/MASK_{self.__id}"

282

283 elif self.__pyramid.storage_type == StorageType.S3:

284 serialization["storage"] = {

285 "type": "S3",

286 "image_prefix": f"{self.__pyramid.name}/DATA_{self.__id}",

287 "bucket_name": self.__pyramid.storage_root,

288 }

289 if self.__pyramid.own_masks:

290 serialization["storage"]["mask_prefix"] = f"{self.__pyramid.name}/MASK_{self.__id}"

291

292 return serialization

293

294 @property

295 def id(self) -> str:

296 return self.__id

297

298 @property

299 def bbox(self) -> Tuple[float, float, float, float]:

300 """Return level extent, based on tile limits

301

302 Returns:

303 Tuple[float, float, float, float]: level terrain extent (xmin, ymin, xmax, ymax)

304 """

305

306 min_bbox = self.__pyramid.tms.get_level(self.__id).tile_to_bbox(

307 self.__tile_limits["min_col"], self.__tile_limits["max_row"]

308 )

309 max_bbox = self.__pyramid.tms.get_level(self.__id).tile_to_bbox(

310 self.__tile_limits["max_col"], self.__tile_limits["min_row"]

311 )

312

313 return (min_bbox[0], min_bbox[1], max_bbox[2], max_bbox[3])

314

315 @property

316 def resolution(self) -> str:

317 return self.__pyramid.tms.get_level(self.__id).resolution

318

319 @property

320 def tile_matrix(self) -> TileMatrix:

321 return self.__pyramid.tms.get_level(self.__id)

322

323 @property

324 def slab_width(self) -> int:

325 return self.__slab_size[0]

326

327 @property

328 def slab_height(self) -> int:

329 return self.__slab_size[1]

330

331 def is_in_limits(self, column: int, row: int) -> bool:

332 """Is the tile indices in limits ?

333

334 Args:

335 column (int): tile's column

336 row (int): tile's row

337

338 Returns:

339 bool: True if tiles' limits contain the provided tile's indices

340 """

341 return (

342 self.__tile_limits["min_row"] <= row

343 and self.__tile_limits["max_row"] >= row

344 and self.__tile_limits["min_col"] <= column

345 and self.__tile_limits["max_col"] >= column

346 )

347

348 def set_limits_from_bbox(self, bbox: Tuple[float, float, float, float]) -> None:

349 """Set tile limits, based on provided bounding box

350

351 Args:

352 bbox (Tuple[float, float, float, float]): terrain extent (xmin, ymin, xmax, ymax), in TMS coordinates system

353

354 """

355

356 col_min, row_min, col_max, row_max = self.__pyramid.tms.get_level(self.__id).bbox_to_tiles(

357 bbox

358 )

359 self.__tile_limits = {

360 "min_row": row_min,

361 "max_col": col_max,

362 "max_row": row_max,

363 "min_col": col_min,

364 }

365

366

367class Pyramid:

368

369 """A data pyramid, raster or vector

370

371 Attributes:

372 __name (str): pyramid's name

373 __descriptor (str): pyramid's descriptor path

374 __list (str): pyramid's list path

375 __tms (rok4.TileMatrixSet.TileMatrixSet): Used grid

376 __levels (Dict[str, Level]): Pyramid's levels

377 __format (str): Data format

378 __storage (Dict[str, Union[rok4.Storage.StorageType,str,int]]): Pyramid's storage informations (type, root and depth if FILE storage)

379 __raster_specifications (Dict): If raster pyramid, raster specifications

380 __content (Dict): Loading status (loaded) and list content (cache).

381

382 Example (S3 storage):

383

384 {

385 'cache': {

386 (<SlabType.DATA: 'DATA'>, '18', 5424, 7526): {

387 'link': False,

388 'md5': None,

389 'root': 'pyramids@localhost:9000/LIMADM',

390 'slab': 'DATA_18_5424_7526'

391 }

392 },

393 'loaded': True

394 }

395 """

396

397 @classmethod

398 def from_descriptor(cls, descriptor: str) -> "Pyramid":

399 """Create a pyramid from its descriptor

400

401 Args:

402 descriptor (str): pyramid's descriptor path

403

404 Raises:

405 FormatError: Provided path or the TMS is not a well formed JSON

406 Exception: Level issue : no one in the pyramid or the used TMS, or level ID not defined in the TMS

407 MissingAttributeError: Attribute is missing in the content

408 StorageError: Storage read issue (pyramid descriptor or TMS)

409 MissingEnvironmentError: Missing object storage informations or TMS root directory

410

411 Examples:

412

413 S3 stored descriptor

414

415 from rok4.Pyramid import Pyramid

416

417 try:

418 pyramid = Pyramid.from_descriptor("s3://bucket_name/path/to/descriptor.json")

419 except Exception as e:

420 print("Cannot load the pyramid from its descriptor")

421

422 Returns:

423 Pyramid: a Pyramid instance

424 """

425 try:

426 data = json.loads(get_data_str(descriptor))

427

428 except JSONDecodeError as e:

429 raise FormatError("JSON", descriptor, e)

430

431 pyramid = cls()

432

433 pyramid.__storage["type"], path, pyramid.__storage["root"], base_name = get_infos_from_path(

434 descriptor

435 )

436 pyramid.__name = base_name[:-5] # on supprime l'extension.json

437 pyramid.__descriptor = descriptor

438 pyramid.__list = get_path_from_infos(

439 pyramid.__storage["type"], pyramid.__storage["root"], f"{pyramid.__name}.list"

440 )

441

442 try:

443 # Attributs communs

444 pyramid.__tms = TileMatrixSet(data["tile_matrix_set"])

445 pyramid.__format = data["format"]

446

447 # Attributs d'une pyramide raster

448 if pyramid.type == PyramidType.RASTER:

449 pyramid.__raster_specifications = data["raster_specifications"]

450

451 if "mask_format" in data:

452 pyramid.__masks = True

453 else:

454 pyramid.__masks = False

455

456 # Niveaux

457 for l in data["levels"]:

458 lev = Level.from_descriptor(l, pyramid)

459 pyramid.__levels[lev.id] = lev

460

461 if pyramid.__tms.get_level(lev.id) is None:

462 raise Exception(

463 f"Pyramid {descriptor} owns a level with the ID '{lev.id}', not defined in the TMS '{pyramid.tms.name}'"

464 )

465

466 except KeyError as e:

467 raise MissingAttributeError(descriptor, e)

468

469 if len(pyramid.__levels.keys()) == 0:

470 raise Exception(f"Pyramid '{descriptor}' has no level")

471

472 return pyramid

473

474 @classmethod

475 def from_other(cls, other: "Pyramid", name: str, storage: Dict) -> "Pyramid":

476 """Create a pyramid from another one

477

478 Args:

479 other (Pyramid): pyramid to clone

480 name (str): new pyramid's name

481 storage (Dict[str, Union[str, int]]): new pyramid's storage informations

482

483 Raises:

484 FormatError: Provided path or the TMS is not a well formed JSON

485 Exception: Level issue : no one in the pyramid or the used TMS, or level ID not defined in the TMS

486 MissingAttributeError: Attribute is missing in the content

487

488 Returns:

489 Pyramid: a Pyramid instance

490 """

491 try:

492 # On convertit le type de stockage selon l'énumération

493 storage["type"] = StorageType[storage["type"]]

494

495 if storage["type"] == StorageType.FILE and name.find("/") != -1:

496 raise Exception(f"A FILE stored pyramid's name cannot contain '/' : '{name}'")

497

498 if storage["type"] == StorageType.FILE and "depth" not in storage:

499 storage["depth"] = 2

500

501 pyramid = cls()

502

503 # Attributs communs

504 pyramid.__name = name

505 pyramid.__storage = storage

506 pyramid.__masks = other.__masks

507

508 pyramid.__descriptor = get_path_from_infos(

509 pyramid.__storage["type"], pyramid.__storage["root"], f"{pyramid.__name}.json"

510 )

511 pyramid.__list = get_path_from_infos(

512 pyramid.__storage["type"], pyramid.__storage["root"], f"{pyramid.__name}.list"

513 )

514 pyramid.__tms = other.__tms

515 pyramid.__format = other.__format

516

517 # Attributs d'une pyramide raster

518 if pyramid.type == PyramidType.RASTER:

519 if other.own_masks:

520 pyramid.__masks = True

521 else:

522 pyramid.__masks = False

523 pyramid.__raster_specifications = other.__raster_specifications

524

525 # Niveaux

526 for l in other.__levels.values():

527 lev = Level.from_other(l, pyramid)

528 pyramid.__levels[lev.id] = lev

529

530 except KeyError as e:

531 raise MissingAttributeError(descriptor, e)

532

533 return pyramid

534

535 def __init__(self) -> None:

536 self.__storage = {}

537 self.__levels = {}

538 self.__masks = None

539

540 self.__content = {"loaded": False, "cache": {}}

541

542 def __str__(self) -> str:

543 return f"{self.type.name} pyramid '{self.__name}' ({self.__storage['type'].name} storage)"

544

545 @property

546 def serializable(self) -> Dict:

547 """Get the dict version of the pyramid object, descriptor compliant

548

549 Returns:

550 Dict: descriptor structured object description

551 """

552

553 serialization = {

554 "tile_matrix_set": self.__tms.name,

555 "format": self.__format

556 }

557

558 serialization["levels"] = []

559 sorted_levels = sorted(self.__levels.values(), key=lambda l: l.resolution, reverse=True)

560

561 for l in sorted_levels:

562 serialization["levels"].append(l.serializable)

563

564 if self.type == PyramidType.RASTER:

565 serialization["raster_specifications"] = self.__raster_specifications

566

567 if self.__masks:

568 serialization["mask_format"] = "TIFF_ZIP_UINT8"

569

570 return serialization

571

572 @property

573 def list(self) -> str:

574 return self.__list

575

576 @property

577 def descriptor(self) -> str:

578 return self.__descriptor

579

580 @property

581 def name(self) -> str:

582 return self.__name

583

584 @property

585 def tms(self) -> TileMatrixSet:

586 return self.__tms

587

588 @property

589 def raster_specifications(self) -> Dict:

590 """Get raster specifications for a RASTER pyramid

591

592 Example:

593 {

594 "channels": 3,

595 "nodata": "255,0,0",

596 "photometric": "rgb",

597 "interpolation": "bicubic"

598 }

599

600 Returns:

601 Dict: Raster specifications, None if VECTOR pyramid

602 """

603 return self.__raster_specifications

604

605 @property

606 def storage_type(self) -> StorageType:

607 """Get the storage type

608

609 Returns:

610 StorageType: FILE, S3 or CEPH

611 """

612 return self.__storage["type"]

613

614 @property

615 def storage_root(self) -> str:

616 """Get the pyramid's storage root.

617

618 If storage is S3, the used cluster is removed.

619

620 Returns:

621 str: Pyramid's storage root

622 """

623

624 return self.__storage["root"].split("@", 1)[

625 0

626 ] # Suppression de l'éventuel hôte de spécification du cluster S3

627

628 @property

629 def storage_depth(self) -> int:

630 return self.__storage.get("depth", None)

631

632 @property

633 def storage_s3_cluster(self) -> str:

634 """Get the pyramid's storage S3 cluster (host name)

635

636 Returns:

637 str: the host if known, None if the default one have to be used or if storage is not S3

638 """

639 if self.__storage["type"] == StorageType.S3:

640 try:

641 return self.__storage["root"].split("@")[1]

642 except IndexError:

643 return None

644 else:

645 return None

646

647 @storage_depth.setter

648 def storage_depth(self, d: int) -> None:

649 """Set the tree depth for a FILE storage

650

651 Args:

652 d (int): file storage depth

653

654 Raises:

655 Exception: the depth is not equal to the already known depth

656 """

657 if "depth" in self.__storage and self.__storage["depth"] != d:

658 raise Exception(

659 f"Pyramid {pyramid.__descriptor} owns levels with different path depths"

660 )

661 self.__storage["depth"] = d

662

663 @property

664 def own_masks(self) -> bool:

665 return self.__masks

666

667 @property

668 def format(self) -> str:

669 return self.__format

670

671 @property

672 def tile_extension(self) -> str:

673

674 if self.__format in [

675 "TIFF_RAW_UINT8",

676 "TIFF_LZW_UINT8",

677 "TIFF_ZIP_UINT8",

678 "TIFF_PKB_UINT8",

679 "TIFF_RAW_FLOAT32",

680 "TIFF_LZW_FLOAT32",

681 "TIFF_ZIP_FLOAT32",

682 "TIFF_PKB_FLOAT32",

683 ]:

684 return "tif"

685 elif self.__format in ["TIFF_JPG_UINT8", "TIFF_JPG90_UINT8"]:

686 return "jpg"

687 elif self.__format == "TIFF_PNG_UINT8":

688 return "png"

689 elif self.__format == "TIFF_PBF_MVT":

690 return "pbf"

691 else:

692 raise Exception(

693 f"Unknown pyramid's format ({self.__format}), cannot return the tile extension"

694 )

695

696 @property

697 def bottom_level(self) -> "Level":

698 """Get the best resolution level in the pyramid

699

700 Returns:

701 Level: the bottom level

702 """

703 return sorted(self.__levels.values(), key=lambda l: l.resolution)[0]

704

705 @property

706 def top_level(self) -> "Level":

707 """Get the low resolution level in the pyramid

708

709 Returns:

710 Level: the top level

711 """

712 return sorted(self.__levels.values(), key=lambda l: l.resolution)[-1]

713

714 @property

715 def type(self) -> PyramidType:

716 """Get the pyramid's type (RASTER or VECTOR) from its format

717

718 Returns:

719 PyramidType: RASTER or VECTOR

720 """

721 if self.__format == "TIFF_PBF_MVT":

722 return PyramidType.VECTOR

723 else:

724 return PyramidType.RASTER

725

726 def load_list(self) -> None:

727 """Load list content and cache it

728

729 If list is already loaded, nothing done

730 """

731 if self.__content["loaded"]:

732 return

733

734 for slab, infos in self.list_generator():

735 self.__content["cache"][slab] = infos

736

737 self.__content["loaded"] = True

738

739 def list_generator(self) -> Iterator[Tuple[Tuple[SlabType, str, int, int], Dict]]:

740 """Get list content

741

742 List is copied as temporary file, roots are read and informations about each slab is returned. If list is already loaded, we yield the cached content

743

744 Examples:

745

746 S3 stored descriptor

747

748 from rok4.Pyramid import Pyramid

749

750 try:

751 pyramid = Pyramid.from_descriptor("s3://bucket_name/path/to/descriptor.json")

752

753 for (slab_type, level, column, row), infos in pyramid.list_generator():

754 print(infos)

755

756 except Exception as e:

757 print("Cannot load the pyramid from its descriptor and read the list")

758

759 Yields:

760 Iterator[Tuple[Tuple[SlabType,str,int,int], Dict]]: Slab indices and storage informations

761

762 Value example:

763

764 (

765 (<SlabType.DATA: 'DATA'>, '18', 5424, 7526),

766 {

767 'link': False,

768 'md5': None,

769 'root': 'pyramids@localhost:9000/LIMADM',

770 'slab': 'DATA_18_5424_7526'

771 }

772 )

773

774 """

775 if self.__content["loaded"]:

776 for slab, infos in self.__content["cache"].items():

777 yield slab, infos

778 else:

779 # Copie de la liste dans un fichier temporaire (cette liste peut être un objet)

780 list_obj = tempfile.NamedTemporaryFile(mode="r", delete=False)

781 list_file = list_obj.name

782 copy(self.__list, f"file://{list_file}")

783 list_obj.close()

784

785 roots = {}

786 s3_cluster = self.storage_s3_cluster

787

788 with open(list_file, "r") as listin:

789 # Lecture des racines

790 for line in listin:

791 line = line.rstrip()

792

793 if line == "#":

794 break

795

796 root_id, root_path = line.split("=", 1)

797

798 if s3_cluster is None:

799 roots[root_id] = root_path

800 else:

801 # On a un nom de cluster S3, on l'ajoute au nom du bucket dans les racines

802 root_bucket, root_path = root_path.split("/", 1)

803 roots[root_id] = f"{root_bucket}@{s3_cluster}/{root_path}"

804

805 # Lecture des dalles

806 for line in listin:

807 line = line.rstrip()

808

809 parts = line.split(" ", 1)

810 slab_path = parts[0]

811 slab_md5 = None

812 if len(parts) == 2:

813 slab_md5 = parts[1]

814

815 root_id, slab_path = slab_path.split("/", 1)

816

817 slab_type, level, column, row = self.get_infos_from_slab_path(slab_path)

818 infos = {

819 "root": roots[root_id],

820 "link": root_id != "0",

821 "slab": slab_path,

822 "md5": slab_md5,

823 }

824

825 yield ((slab_type, level, column, row), infos)

826

827 remove(f"file://{list_file}")

828

829 def get_level(self, level_id: str) -> "Level":

830 """Get one level according to its identifier

831

832 Args:

833 level_id: Level identifier

834

835 Returns:

836 The corresponding pyramid's level, None if not present

837 """

838

839 return self.__levels.get(level_id, None)

840

841 def get_levels(self, bottom_id: str = None, top_id: str = None) -> List[Level]:

842 """Get sorted levels in the provided range from bottom to top

843

844 Args:

845 bottom_id (str, optionnal): specific bottom level id. Defaults to None.

846 top_id (str, optionnal): specific top level id. Defaults to None.

847

848 Raises:

849 Exception: Provided levels are not consistent (bottom > top or not in the pyramid)

850

851 Examples:

852

853 All levels

854

855 from rok4.Pyramid import Pyramid

856

857 try:

858 pyramid = Pyramid.from_descriptor("s3://bucket_name/path/to/descriptor.json")

859 levels = pyramid.get_levels()

860

861 except Exception as e:

862 print("Cannot load the pyramid from its descriptor and get levels")

863

864 From pyramid's bottom to provided top (level 5)

865

866 from rok4.Pyramid import Pyramid

867

868 try:

869 pyramid = Pyramid.from_descriptor("s3://bucket_name/path/to/descriptor.json")

870 levels = pyramid.get_levels(None, "5")

871

872 except Exception as e:

873 print("Cannot load the pyramid from its descriptor and get levels")

874

875 Returns:

876 List[Level]: asked sorted levels

877 """

878

879 sorted_levels = sorted(self.__levels.values(), key=lambda l: l.resolution)

880

881 levels = []

882

883 begin = False

884 if bottom_id is None:

885 # Pas de niveau du bas fourni, on commence tout en bas

886 begin = True

887 else:

888 if self.get_level(bottom_id) is None:

889 raise Exception(

890 f"Pyramid {self.name} does not contain the provided bottom level {bottom_id}"

891 )

892

893 if top_id is not None and self.get_level(top_id) is None:

894 raise Exception(f"Pyramid {self.name} does not contain the provided top level {top_id}")

895

896 end = False

897

898 for l in sorted_levels:

899 if not begin and l.id == bottom_id:

900 begin = True

901

902 if begin:

903 levels.append(l)

904 if top_id is not None and l.id == top_id:

905 end = True

906 break

907 else:

908 continue

909

910 if top_id is None:

911 # Pas de niveau du haut fourni, on a été jusqu'en haut et c'est normal

912 end = True

913

914 if not begin or not end:

915 raise Exception(

916 f"Provided levels ids are not consistent to extract levels from the pyramid {self.name}"

917 )

918

919 return levels

920

921 def write_descriptor(self) -> None:

922 """Write the pyramid's descriptor to the final location (in the pyramid's storage root)"""

923

924 content = json.dumps(self.serializable)

925 put_data_str(content, self.__descriptor)

926

927 def get_infos_from_slab_path(self, path: str) -> Tuple[SlabType, str, int, int]:

928 """Get the slab's indices from its storage path

929

930 Args:

931 path (str): Slab's storage path

932

933 Examples:

934

935 FILE stored pyramid

936

937 from rok4.Pyramid import Pyramid

938

939 try:

940 pyramid = Pyramid.from_descriptor("/path/to/descriptor.json")

941 slab_type, level, column, row = self.get_infos_from_slab_path("DATA/12/00/4A/F7.tif")

942 # (SlabType.DATA, "12", 159, 367)

943 except Exception as e:

944 print("Cannot load the pyramid from its descriptor and convert a slab path")

945

946 S3 stored pyramid

947

948 from rok4.Pyramid import Pyramid

949

950 try:

951 pyramid = Pyramid.from_descriptor("s3://bucket_name/path/to/pyramid.json")

952 slab_type, level, column, row = self.get_infos_from_slab_path("s3://bucket_name/path/to/pyramid/MASK_15_9164_5846")

953 # (SlabType.MASK, "15", 9164, 5846)

954 except Exception as e:

955 print("Cannot load the pyramid from its descriptor and convert a slab path")

956

957 Returns:

958 Tuple[SlabType, str, int, int]: Slab's type (DATA or MASK), level identifier, slab's column and slab's row

959 """

960 if self.__storage["type"] == StorageType.FILE:

961 parts = path.split("/")

962

963 # Le partie du chemin qui contient la colonne et ligne de la dalle est à la fin, en fonction de la profondeur choisie

964 # depth = 2 -> on doit utiliser les 3 dernières parties pour la conversion

965 column, row = b36_path_decode("/".join(parts[-(self.__storage["depth"] + 1) :]))

966 level = parts[-(self.__storage["depth"] + 2)]

967 raw_slab_type = parts[-(self.__storage["depth"] + 3)]

968

969 # Pour être retro compatible avec l'ancien nommage

970 if raw_slab_type == "IMAGE":

971 raw_slab_type = "DATA"

972

973 slab_type = SlabType[raw_slab_type]

974

975 return slab_type, level, column, row

976 else:

977 parts = re.split(r"[/_]", path)

978 column = parts[-2]

979 row = parts[-1]

980 level = parts[-3]

981 raw_slab_type = parts[-4]

982

983 # Pour être retro compatible avec l'ancien nommage

984 if raw_slab_type == "IMG":

985 raw_slab_type = "DATA"

986 elif raw_slab_type == "MSK":

987 raw_slab_type = "MASK"

988

989 slab_type = SlabType[raw_slab_type]

990

991 return slab_type, level, int(column), int(row)

992

993 def get_slab_path_from_infos(

994 self, slab_type: SlabType, level: str, column: int, row: int, full: bool = True

995 ) -> str:

996 """Get slab's storage path from the indices

997

998 Args:

999 slab_type (SlabType): DATA or MASK

1000 level (str): Level identifier

1001 column (int): Slab's column

1002 row (int): Slab's row

1003 full (bool, optional): Full path or just relative path from pyramid storage root. Defaults to True.

1004

1005 Returns:

1006 str: Absolute or relative slab's storage path

1007 """

1008 if self.__storage["type"] == StorageType.FILE:

1009 slab_path = os.path.join(

1010 slab_type.value, level, b36_path_encode(column, row, self.__storage["depth"])

1011 )

1012 else:

1013 slab_path = f"{slab_type.value}_{level}_{column}_{row}"

1014

1015 if full:

1016 return get_path_from_infos(

1017 self.__storage["type"], self.__storage["root"], self.__name, slab_path

1018 )

1019 else:

1020 return slab_path

1021

1022

1023 def get_tile_data_binary(self, level: str, column: int, row: int) -> str:

1024 """Get a pyramid's tile as binary string

1025

1026 To get a tile, 3 steps :

1027 * calculate slab path from tile index

1028 * read slab index to get offsets and sizes of slab's tiles

1029 * read the tile into the slab

1030

1031 Args:

1032 level (str): Tile's level

1033 column (int): Tile's column

1034 row (int): Tile's row

1035

1036 Limitations:

1037 Pyramids with one-tile slab are not handled

1038

1039 Examples:

1040

1041 FILE stored raster pyramid, to extract a tile containing a point and save it as independent image

1042

1043 from rok4.Pyramid import Pyramid

1044

1045 try:

1046 pyramid = Pyramid.from_descriptor("/data/pyramids/SCAN1000.json")

1047 level, col, row, pcol, prow = pyramid.get_tile_indices(992904.46, 6733643.15, "9", srs = "IGNF:LAMB93")

1048 data = pyramid.get_tile_data_binary(level, col, row)

1049

1050 if data is None:

1051 print("No data")

1052 else:

1053 tile_name = f"tile_{level}_{col}_{row}.{pyramid.tile_extension}"

1054 with open(tile_name, "wb") as image:

1055 image.write(data)

1056 print (f"Tile written in {tile_name}")

1057

1058 except Exception as e:

1059 print("Cannot save a pyramid's tile : {e}")

1060

1061 Raises:

1062 Exception: Level not found in the pyramid

1063 NotImplementedError: Pyramid owns one-tile slabs

1064 MissingEnvironmentError: Missing object storage informations

1065 StorageError: Storage read issue

1066

1067 Returns:

1068 str: data, as binary string, None if no data

1069 """

1070

1071 level_object = self.get_level(level)

1072

1073 if level_object is None:

1074 raise Exception(f"No level {level} in the pyramid")

1075

1076 if level_object.slab_width == 1 and level_object.slab_height == 1:

1077 raise NotImplementedError(f"One-tile slab pyramid is not handled")

1078

1079 if not level_object.is_in_limits(column, row):

1080 return None

1081

1082 # Indices de la dalle

1083 slab_column = column // level_object.slab_width

1084 slab_row = row // level_object.slab_height

1085

1086 # Indices de la tuile dans la dalle

1087 relative_tile_column = column % level_object.slab_width

1088 relative_tile_row = row % level_object.slab_height

1089

1090 # Numéro de la tuile dans le header

1091 tile_index = relative_tile_row * level_object.slab_width + relative_tile_column

1092

1093 # Calcul du chemin de la dalle contenant la tuile voulue

1094 slab_path = self.get_slab_path_from_infos(SlabType.DATA, level, slab_column, slab_row)

1095

1096 # Récupération des offset et tailles des tuiles dans la dalle

1097 # Une dalle ROK4 a une en-tête fixe de 2048 octets,

1098 # puis sont stockés les offsets (chacun sur 4 octets)

1099 # puis les tailles (chacune sur 4 octets)

1100 try:

1101 binary_index = get_data_binary(

1102 slab_path,

1103 (

1104 ROK4_IMAGE_HEADER_SIZE,

1105 2 * 4 * level_object.slab_width * level_object.slab_height,

1106 ),

1107 )

1108 except FileNotFoundError as e:

1109 # L'absence de la dalle est gérée comme simplement une absence de données

1110 return None

1111

1112 offsets = numpy.frombuffer(

1113 binary_index,

1114 dtype=numpy.dtype("uint32"),

1115 count=level_object.slab_width * level_object.slab_height,

1116 )

1117 sizes = numpy.frombuffer(

1118 binary_index,

1119 dtype=numpy.dtype("uint32"),

1120 offset=4 * level_object.slab_width * level_object.slab_height,

1121 count=level_object.slab_width * level_object.slab_height,

1122 )

1123

1124 if sizes[tile_index] == 0:

1125 return None

1126

1127 return get_data_binary(slab_path, (offsets[tile_index], sizes[tile_index]))

1128

1129 def get_tile_data_raster(self, level: str, column: int, row: int) -> numpy.ndarray:

1130 """Get a raster pyramid's tile as 3-dimension numpy ndarray

1131

1132 First dimension is the row, second one is column, third one is band.

1133

1134 Args:

1135 level (str): Tile's level

1136 column (int): Tile's column

1137 row (int): Tile's row

1138

1139 Limitations:

1140 Packbits (pyramid formats TIFF_PKB_FLOAT32 and TIFF_PKB_UINT8) and LZW (pyramid formats TIFF_LZW_FLOAT32 and TIFF_LZW_UINT8) compressions are not handled.

1141

1142 Raises:

1143 Exception: Cannot get raster data for a vector pyramid

1144 Exception: Level not found in the pyramid

1145 NotImplementedError: Pyramid owns one-tile slabs

1146 NotImplementedError: Raster pyramid format not handled

1147 MissingEnvironmentError: Missing object storage informations

1148 StorageError: Storage read issue

1149 FormatError: Cannot decode tile

1150

1151 Examples:

1152

1153 FILE stored DTM (raster) pyramid, to get the altitude value at a point in the best level

1154

1155 from rok4.Pyramid import Pyramid

1156

1157 try:

1158 pyramid = Pyramid.from_descriptor("/data/pyramids/RGEALTI.json")

1159 level, col, row, pcol, prow = pyramid.get_tile_indices(44, 5, srs = "EPSG:4326")

1160 data = pyramid.get_tile_data_raster(level, col, row)

1161

1162 if data is None:

1163 print("No data")

1164 else:

1165 print(data[prow][pcol])

1166

1167 except Exception as e:

1168 print("Cannot get a pyramid's pixel value : {e}")

1169

1170 Returns:

1171 str: data, as numpy array, None if no data

1172 """

1173

1174 if self.type == PyramidType.VECTOR:

1175 raise Exception("Cannot get tile as raster data : it's a vector pyramid")

1176

1177 binary_tile = self.get_tile_data_binary(level, column, row)

1178

1179 if binary_tile is None:

1180 return None

1181

1182 level_object = self.get_level(level)

1183

1184 if self.__format == "TIFF_JPG_UINT8" or self.__format == "TIFF_JPG90_UINT8":

1185

1186 try:

1187 img = Image.open(io.BytesIO(binary_tile))

1188 except Exception as e:

1189 raise FormatError("JPEG", "binary tile", e)

1190

1191 data = numpy.asarray(img)

1192

1193 elif self.__format == "TIFF_RAW_UINT8":

1194 data = numpy.frombuffer(binary_tile, dtype=numpy.dtype("uint8"))

1195 data.shape = (

1196 level_object.tile_matrix.tile_size[0],

1197 level_object.tile_matrix.tile_size[1],

1198 self.__raster_specifications["channels"],

1199 )

1200

1201 elif self.__format == "TIFF_PNG_UINT8":

1202 try:

1203 img = Image.open(io.BytesIO(binary_tile))

1204 except Exception as e:

1205 raise FormatError("PNG", "binary tile", e)

1206

1207 data = numpy.asarray(img)

1208

1209 elif self.__format == "TIFF_ZIP_UINT8":

1210 try:

1211 data = numpy.frombuffer(zlib.decompress(binary_tile), dtype=numpy.dtype("uint8"))

1212 except Exception as e:

1213 raise FormatError("ZIP", "binary tile", e)

1214

1215 data.shape = (

1216 level_object.tile_matrix.tile_size[0],

1217 level_object.tile_matrix.tile_size[1],

1218 self.__raster_specifications["channels"],

1219 )

1220

1221 elif self.__format == "TIFF_ZIP_FLOAT32":

1222 try:

1223 data = numpy.frombuffer(zlib.decompress(binary_tile), dtype=numpy.dtype("float32"))

1224 except Exception as e:

1225 raise FormatError("ZIP", "binary tile", e)

1226

1227 data.shape = (

1228 level_object.tile_matrix.tile_size[0],

1229 level_object.tile_matrix.tile_size[1],

1230 self.__raster_specifications["channels"],

1231 )

1232

1233 elif self.__format == "TIFF_RAW_FLOAT32":

1234 data = numpy.frombuffer(binary_tile, dtype=numpy.dtype("float32"))

1235 data.shape = (

1236 level_object.tile_matrix.tile_size[0],

1237 level_object.tile_matrix.tile_size[1],

1238 self.__raster_specifications["channels"],

1239 )

1240

1241 else:

1242 raise NotImplementedError(f"Cannot get tile as raster data for format {self.__format}")

1243

1244 return data

1245

1246 def get_tile_data_vector(self, level: str, column: int, row: int) -> Dict:

1247 """Get a vector pyramid's tile as GeoJSON dictionnary

1248

1249 Args:

1250 level (str): Tile's level

1251 column (int): Tile's column

1252 row (int): Tile's row

1253

1254 Raises:

1255 Exception: Cannot get vector data for a raster pyramid

1256 Exception: Level not found in the pyramid

1257 NotImplementedError: Pyramid owns one-tile slabs

1258 NotImplementedError: Vector pyramid format not handled

1259 MissingEnvironmentError: Missing object storage informations

1260 StorageError: Storage read issue

1261 FormatError: Cannot decode tile

1262

1263 Examples:

1264

1265 S3 stored vector pyramid, to print a tile as GeoJSON

1266

1267 from rok4.Pyramid import Pyramid

1268 import json

1269

1270 try:

1271 pyramid = Pyramid.from_descriptor("s3://pyramids/vectors/BDTOPO.json")

1272 level, col, row, pcol, prow = pyramid.get_tile_indices(40.325, 3.123, srs = "EPSG:4326")

1273 data = pyramid.get_tile_data_vector(level, col, row)

1274

1275 if data is None:

1276 print("No data")

1277 else:

1278 print(json.dumps(data))

1279

1280 except Exception as e:

1281 print("Cannot print a vector pyramid's tile as GeoJSON : {e}")

1282

1283 Returns:

1284 str: data, as GeoJSON dictionnary. None if no data

1285 """

1286

1287 if self.type == PyramidType.RASTER:

1288 raise Exception("Cannot get tile as vector data : it's a raster pyramid")

1289

1290 binary_tile = self.get_tile_data_binary(level, column, row)

1291

1292 if binary_tile is None:

1293 return None

1294

1295 level_object = self.get_level(level)

1296

1297 if self.__format == "TIFF_PBF_MVT":

1298 try:

1299 data = mapbox_vector_tile.decode(binary_tile)

1300 except Exception as e:

1301 raise FormatError("PBF (MVT)", "binary tile", e)

1302 else:

1303 raise NotImplementedError(f"Cannot get tile as vector data for format {self.__format}")

1304

1305 return data

1306

1307 def get_tile_indices(

1308 self, x: float, y: float, level: str = None, **kwargs

1309 ) -> Tuple[str, int, int, int, int]:

1310 """Get pyramid's tile and pixel indices from point's coordinates

1311

1312 Used coordinates system have to be the pyramid one. If EPSG:4326, x is latitude and y longitude.

1313

1314 Args:

1315 x (float): point's x

1316 y (float): point's y

1317 level (str, optional): Pyramid's level to take into account, the bottom one if None . Defaults to None.

1318 **srs (string): spatial reference system of provided coordinates, with authority and code (same as the pyramid's one if not provided)

1319

1320 Raises:

1321 Exception: Cannot find level to calculate indices

1322 RuntimeError: Provided SRS is invalid for OSR

1323

1324 Examples:

1325

1326 FILE stored DTM (raster) pyramid, to get the altitude value at a point in the best level

1327

1328 from rok4.Pyramid import Pyramid

1329

1330 try:

1331 pyramid = Pyramid.from_descriptor("/data/pyramids/RGEALTI.json")

1332 level, col, row, pcol, prow = pyramid.get_tile_indices(44, 5, srs = "EPSG:4326")

1333 data = pyramid.get_tile_data_raster(level, col, row)

1334

1335 if data is None:

1336 print("No data")

1337 else:

1338 print(data[prow][pcol])

1339

1340 except Exception as e:

1341 print("Cannot get a pyramid's pixel value : {e}")

1342

1343 Returns:

1344 Tuple[str, int, int, int, int]: Level identifier, tile's column, tile's row, pixel's (in the tile) column, pixel's row

1345 """

1346

1347 level_object = self.bottom_level

1348 if level is not None:

1349 level_object = self.get_level(level)

1350

1351 if level_object is None:

1352 raise Exception(f"Cannot found the level to calculate indices")

1353

1354 if (

1355 "srs" in kwargs

1356 and kwargs["srs"] is not None

1357 and kwargs["srs"].upper() != self.__tms.srs.upper()

1358 ):

1359 sr = srs_to_spatialreference(kwargs["srs"])

1360 x, y = reproject_point((x, y), sr, self.__tms.sr)

1361

1362 return (level_object.id,) + level_object.tile_matrix.point_to_indices(x, y)

1363

1364 @property

1365 def size(self) -> int:

1366 """Get the size of the pyramid

1367

1368 Examples:

1369

1370 from rok4.Pyramid import Pyramid

1371

1372 try:

1373 pyramid = Pyramid.from_descriptor("s3://bucket_name/path/to/descriptor.json")

1374 size = pyramid.size()

1375

1376 except Exception as e:

1377 print("Cannot load the pyramid from its descriptor and get his size")

1378

1379 Returns:

1380 int: size of the pyramid

1381 """

1382 if not hasattr(self,"_Pyramid__size") :

1383 self.__size = size_path(get_path_from_infos(self.__storage["type"], self.__storage["root"], self.__name))

1384 return self.__size