diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 203b789ac2485afeb0b65490aaad3be1a05dd177..57c9c42d389910f0120fd827788b30637f7c0398 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,16 +1,29 @@ -workflow: - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_REF_NAME =~ /master/ - default: image: $CI_REGISTRY/orfeotoolbox/otb-build-env/otb-ubuntu-native-develop-headless:20.04 tags: - light + interruptible: true + +variables: + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + +cache: + key: $CI_COMMIT_REF_SLUG + paths: + - .cache/pip + +workflow: + rules: + - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" + when: never + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_TAG + - if: $CI_COMMIT_REF_PROTECTED == "true" stages: - Static Analysis - - Documentation test - Tests + - Documentation - Ship # -------------------------------- Static analysis -------------------------------- @@ -18,117 +31,121 @@ stages: .static_analysis: stage: Static Analysis allow_failure: true - -codespell: - extends: .static_analysis - before_script: - - pip install codespell - script: - - codespell {pyotb,tests} - -flake8: - extends: .static_analysis - before_script: - - pip install flake8 - script: - - flake8 --max-line-length=120 $PWD/pyotb --ignore=F403,E402,F401,W503,W504 + rules: + - changes: + - pyotb/*.py pydocstyle: extends: .static_analysis before_script: - - pip install pydocstyle + - pip install pydocstyle tomli script: - - pydocstyle $PWD/pyotb --convention=google + - pydocstyle $PWD/pyotb pylint: extends: .static_analysis before_script: - pip install pylint script: - - pylint --max-line-length=120 $PWD/pyotb --disable=too-many-nested-blocks,too-many-locals,too-many-statements,too-few-public-methods,too-many-instance-attributes,too-many-arguments,invalid-name,fixme,too-many-return-statements,too-many-lines,too-many-branches,import-outside-toplevel,wrong-import-position,wrong-import-order,import-error,missing-class-docstring - -# ---------------------------------- Documentation ---------------------------------- + - pylint $PWD/pyotb --disable=fixme -.docs: +codespell: + extends: .static_analysis + rules: + - changes: + - "**/*.py" + - "**/*.md" before_script: - - apt-get update && apt-get -y install virtualenv - - virtualenv doc_env - - source doc_env/bin/activate - - pip install -U pip - - pip install -U -r doc/doc_requirements.txt - -pages_test: - stage: Documentation test - extends: .docs - except: - - master - when: manual + - pip install codespell script: - - mkdocs build --site-dir public_test - artifacts: - paths: - - public_test + - codespell {pyotb,tests,doc,README.md} -# -------------------------------------- Tests -------------------------------------- +# -------------------------------------- Tests -------------------------------------- +test_install: + stage: Tests + only: + - tags + allow_failure: false + script: + - pip install . .tests: stage: Tests allow_failure: false + rules: + - changes: + - "**/*.py" + - .gitlab-ci.yml variables: OTB_ROOT: /opt/otb LD_LIBRARY_PATH: /opt/otb/lib + SPOT_IMG_URL: https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif + PLEIADES_IMG_URL: https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Baseline/OTB/Images/prTvOrthoRectification_pleiades-1_noDEM.tif + before_script: + - pip install pytest pytest-cov + +module_core: + extends: .tests + variables: OTB_LOGGER_LEVEL: INFO PYOTB_LOGGER_LEVEL: DEBUG - IMAGE_URL: https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif?inline=false - TEST_INPUT_IMAGE: tests/image.tif artifacts: reports: - junit: test-*.xml - before_script: - - wget $IMAGE_URL -O $TEST_INPUT_IMAGE - - pip install pytest - -test_core: - extends: .tests - script: - - python3 -m pytest --color=yes --junitxml=test-core.xml tests/test_core.py - -test_numpy: - extends: .tests + junit: test-module-core.xml + coverage_report: + coverage_format: cobertura + path: coverage.xml + coverage: '/TOTAL.*\s+(\d+%)$/' script: - - python3 -m pytest --color=yes --junitxml=test-numpy.xml tests/test_numpy.py + - curl -fsLI $SPOT_IMG_URL + - curl -fsLI $PLEIADES_IMG_URL + - python3 -m pytest -vv --junitxml=test-module-core.xml --cov-report xml:coverage.xml tests/test_core.py -test_pipeline: - #when: manual - extends: .tests - script: - - python3 -m pytest --color=yes --junitxml=test-pipeline.xml tests/test_pipeline.py - -test_serialization: +pipeline_permutations: extends: .tests + variables: + OTB_LOGGER_LEVEL: WARNING + PYOTB_LOGGER_LEVEL: INFO + artifacts: + reports: + junit: test-pipeline-permutations.xml script: - - python3 -m pytest --color=yes --junitxml=test-serialization.xml tests/test_serialization.py + - curl -fsLI $SPOT_IMG_URL + - python3 -m pytest -vv --junitxml=test-pipeline-permutations.xml tests/test_pipeline.py -# -------------------------------------- Ship --------------------------------------- +# -------------------------------------- Docs --------------------------------------- -pages: - stage: Ship - extends: .docs - only: - - master +docs: + stage: Documentation + rules: + - changes: + - "*.txt" + - "*.md" + - mkdocs.yml + - doc/* + - pyotb/*.py + before_script: + - apt update && apt install -y python3.8-venv + - python3 -m venv docs_venv + - source docs_venv/bin/activate + - python3 -m pip install -U pip + - python3 -m pip install -r doc/doc_requirements.txt script: - mkdocs build --site-dir public artifacts: paths: - public +# -------------------------------------- Ship --------------------------------------- + pypi: stage: Ship + # when: manual only: - - master + - tags before_script: - - apt update && apt install -y python3.8-venv - - python3 -m pip install --upgrade build twine + - apt update && apt install -y python3.8-venv + - pip install build twine script: - - python3 -m build - - python3 -m twine upload --repository-url https://upload.pypi.org/legacy/ --non-interactive -u __token__ -p $pypi_token dist/* + - python3 -m build + - python3 -m twine upload --non-interactive -u __token__ -p $pypi_token dist/* diff --git a/AUTHORS.md b/AUTHORS.md new file mode 100644 index 0000000000000000000000000000000000000000..6518258b14e24c64202014b1015840145fcc644b --- /dev/null +++ b/AUTHORS.md @@ -0,0 +1,10 @@ +# Project authors + +## Initial codebase + +* Nicolas NARÇON (INRAE, now ESA) + +## Current maintainers + +* Rémi CRESSON (INRAE) +* Vincent DELBAR (La TeleScop) diff --git a/README.md b/README.md index 60659a86f5589eb45d2b55291fa5357698becb11..f5aa1841c554fd75311e815b95dd4dd3a5d2892a 100644 --- a/README.md +++ b/README.md @@ -1,318 +1,48 @@ -# pyotb: a pythonic extension of OTB +# pyotb: Orfeo ToolBox for Python -Full documentation is available at [pyotb.readthedocs.io](https://pyotb.readthedocs.io/) +[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/releases) +[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/commits/develop) +[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/commits/develop) +[](https://pyotb.readthedocs.io/en/master/) -[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/releases) -[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/commits/master) +**pyotb** wraps the [Orfeo Toolbox](https://www.orfeo-toolbox.org/) in a pythonic, developer friendly +fashion. +## Key features -## Installation -Requirements: -- python>=3.5 and numpy -- OrfeoToolBox python API +- Easy use of Orfeo ToolBox (OTB) applications from python +- Simplify common sophisticated I/O features of OTB +- Lazy execution of operations thanks to OTB streaming mechanism +- Interoperable with popular python libraries ([numpy](https://numpy.org/) and +[rasterio](https://rasterio.readthedocs.io/)) +- Extensible -```bash -pip install pyotb --upgrade -``` - -For Python>=3.6, latest version available is pyotb 1.5.1 For Python 3.5, latest version available is pyotb 1.2.2 - -## Quickstart: running an OTB application as a oneliner -pyotb has been written so that it is more convenient to run an application in Python. - -You can pass the parameters of an application as a dictionary : -```python -import pyotb -resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'interpolator': 'linear', - 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5}) -``` -Note that pyotb has a 'lazy' evaluation: it only performs operation when it is needed, i.e. results are written to disk. -Thus, the previous line doesn't trigger the application. - -To actually trigger the application execution, you need to write the result to disk: - -```python -resampled.write('output.tif') # this is when the application actually runs -``` - -## Using Python keyword arguments -It is also possible to use the Python keyword arguments notation for passing the parameters: -```python -output = pyotb.SuperImpose(inr='reference_image.tif', inm='image.tif') -``` -is equivalent to: -```python -output = pyotb.SuperImpose({'inr': 'reference_image.tif', 'inm': 'image.tif'}) -``` - -Limitations : for this notation, python doesn't accept the parameter `in` or any parameter that contains a `.`. E.g., it is not possible to use `pyotb.RigidTransformResample(in=input_path...)` or `pyotb.VectorDataExtractROI(io.vd=vector_path...)`. - - - - -## In-memory connections -The big asset of pyotb is the ease of in-memory connections between apps. - -Let's start from our previous example. Consider the case where one wants to apply optical calibration and binary morphological dilatation -following the undersampling. - -Using pyotb, you can pass the output of an app as input of another app : -```python -import pyotb - -resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'interpolator': 'linear', - 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5}) - -calibrated = pyotb.OpticalCalibration({'in': resampled, 'level': 'toa'}) - -dilated = pyotb.BinaryMorphologicalOperation({'in': calibrated, 'out': 'output.tif', 'filter': 'dilate', - 'structype': 'ball', 'xradius': 3, 'yradius': 3}) -dilated.write('result.tif') -``` - -## Writing the result of an app -Any pyotb object can be written to disk using the `write` method, e.g. : - -```python -import pyotb - -resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'interpolator': 'linear', - 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5}) -# Here you can set optionally pixel type and extended filename variables -resampled.write({'out': 'output.tif'}, pixel_type='uint16', filename_extension='?nodata=65535') -``` - -Another possibility for writing results is to set the output parameter when initializing the application: -```python -import pyotb - -resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'interpolator': 'linear', 'out': 'output.tif', - 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5}) -# Here you can set optionally pixel type and extended filename variables -resampled.write(pixel_type='uint16', filename_extension='?nodata=65535') -``` - -## Arithmetic operations -Every pyotb object supports arithmetic operations, such as addition, subtraction, comparison... -Consider an example where we want to compute a vegeteation mask from NDVI, i.e. the arithmetic operation `(nir - red) / (nir + red) > 0.3` - -With pyotb, one can simply do : -```python -import pyotb - -# transforming filepaths to pyotb objects -nir, red = pyotb.Input('nir.tif'), pyotb.Input('red.tif') - -res = (nir - red) / (nir + red) > 0.3 -print(res.exp) # prints the BandMath expression: "((im1b1 - im2b1) / (im1b1 + im2b1)) > 0.3 ? 1 : 0" -res.write('vegetation_mask.tif', pixel_type='uint8') -``` - -## Slicing -pyotb objects support slicing in a Python fashion : - -```python -import pyotb +Documentation hosted at [pyotb.readthedocs.io](https://pyotb.readthedocs.io/). -# transforming filepath to pyotb object -inp = pyotb.Input('my_image.tif') +## Example -inp[:, :, :3] # selecting first 3 bands -inp[:, :, [0, 1, 4]] # selecting bands 1, 2 & 5 -inp[:1000, :1000] # selecting 1000x1000 subset, same as inp[:1000, :1000, :] -inp[:100, :100].write('my_image_roi.tif') # write cropped image to disk -``` - -## Numpy-inspired functions -Some functions have been written, entirely based on OTB, to mimic the behavior of some well-known numpy functions. -### pyotb.where -Equivalent of `numpy.where`. -It is the equivalent of the muparser syntax `condition ? x : y` that can be used in OTB's BandMath. - -```python -import pyotb - -# transforming filepaths to pyotb objects -labels, image1, image2 = pyotb.Input('labels.tif'), pyotb.Input('image1.tif') , pyotb.Input('image2.tif') - -# If labels = 1, returns image1. Else, returns image2 -res = pyotb.where(labels == 1, image1, image2) # this would also work: pyotb.where(labels == 1, 'image1.tif', 'image2.tif') - -# A more complex example -# If labels = 1, returns image1. If labels = 2, returns image2. If labels = 3, returns 3. Else 0 -res = pyotb.where(labels == 1, image1, - pyotb.where(labels == 2, image2, - pyotb.where(labels == 3, 3, 0))) +Building a simple pipeline with OTB applications -``` - -### pyotb.clip -Equivalent of `numpy.clip`. Clip (limit) the values in a raster to a range. - -```python +```py import pyotb -res = pyotb.clip('my_image.tif', 0, 255) # clips the values between 0 and 255 -``` - -### pyotb.all -Equivalent of `numpy.all`. - -For only one image, this function checks that all bands of the image are True (i.e. !=0) and outputs -a singleband boolean raster. -For several images, this function checks that all images are True (i.e. !=0) and outputs -a boolean raster, with as many bands as the inputs. - - -### pyotb.any -Equivalent of `numpy.any`. - -For only one image, this function checks that at least one band of the image is True (i.e. !=0) and outputs -a singleband boolean raster. -For several images, this function checks that at least one of the images is True (i.e. !=0) and outputs -a boolean raster, with as many bands as the inputs. - - -## Interaction with Numpy - -pyotb objects can be transparently used in numpy functions. - -For example: - -```python -import pyotb -import numpy as np - -inp = pyotb.Input('image.tif') # this is a pyotb object - -# Creating a numpy array of noise -white_noise = np.random.normal(0, 50, size=inp.shape) # this is a numpy object - -# Adding the noise to the image -noisy_image = inp + white_noise # magic: this is a pyotb object that has the same georeference as input. - # `np.add(inp, white_noise)` would have worked the same -noisy_image.write('image_plus_noise.tif') -``` -Limitations : -- The whole image is loaded into memory -- The georeference can not be modified. Thus, numpy operations can not change the image or pixel size - - -## Export to rasterio -pyotb objects can also be exported in a format that is usable by rasterio. - -For example: - -```python -import pyotb -import rasterio -from scipy import ndimage - -# Pansharpening + NDVI + creating bare soils mask -pxs = pyotb.BundleToPerfectSensor(inp='panchromatic.tif', inxs='multispectral.tif') -ndvi = pyotb.RadiometricIndices({'in': pxs, 'channels.red': 3, 'channels.nir': 4, 'list': 'Vegetation:NDVI'}) -bare_soil_mask = (ndvi < 0.3) - -# Exporting the result as array & profile usable by rasterio -mask_array, profile = bare_soil_mask.to_rasterio() - -# Doing something in Python that is not possible with OTB, e.g. gathering the contiguous groups of pixels -# with an integer index -labeled_mask_array, nb_groups = ndimage.label(mask_array) - -# Writing the result to disk -with rasterio.open('labeled_bare_soil.tif', 'w', **profile) as f: - f.write(labeled_mask_array) - -``` -This way of exporting pyotb objects is more flexible that exporting to numpy, as the user gets the `profile` dictionary. -If the georeference or pixel size is modified, the user can update the `profile` accordingly. - - -## Interaction with Tensorflow - -We saw that numpy operations had some limitations. To bypass those limitations, it is possible to use some Tensorflow operations on pyotb objects. - - -You need a working installation of OTBTF >=3.0 for this and then the code is like this: - -```python -import pyotb - -def scalar_product(x1, x2): - """This is a function composed of tensorflow operations.""" - import tensorflow as tf - return tf.reduce_sum(tf.multiply(x1, x2), axis=-1) - -# Compute the scalar product -res = pyotb.run_tf_function(scalar_product)('image1.tif', 'image2.tif') # magic: this is a pyotb object -res.write('scalar_product.tif') -``` - -For some easy syntax, one can use `pyotb.run_tf_function` as a function decorator, such as: -```python -import pyotb - -@pyotb.run_tf_function # The decorator enables the use of pyotb objects as inputs/output of the function -def scalar_product(x1, x2): - import tensorflow as tf - return tf.reduce_sum(tf.multiply(x1, x2), axis=-1) - -res = scalar_product('image1.tif', 'image2.tif') # magic: this is a pyotb object -``` - -Advantages : -- The process supports streaming, hence the whole image is **not** loaded into memory -- Can be integrated in OTB pipelines - -Limitations : -- It is not possible to use the tensorflow python API inside a script where OTBTF is used because of compilation issues -between Tensorflow and OTBTF, i.e. `import tensorflow` doesn't work in a script where OTBTF apps have been initialized - - -## Some examples -### Compute the mean of several rasters, taking into account NoData -Let's consider we have at disposal 73 NDVI rasters for a year, where clouds have been masked with NoData (nodata value of -10 000 for example). - -Goal: compute the mean across time (keeping the spatial dimension) of the NDVIs, excluding cloudy pixels. Piece of code to achieve that: - -```python -import pyotb - -nodata = -10000 -ndvis = [pyotb.Input(path) for path in ndvi_paths] - -# For each pixel location, summing all valid NDVI values -summed = sum([pyotb.where(ndvi != nodata, ndvi, 0) for ndvi in ndvis]) - -# Printing the generated BandMath expression -print(summed.exp) # this returns a very long exp: "0 + ((im1b1 != -10000) ? im1b1 : 0) + ((im2b1 != -10000) ? im2b1 : 0) + ... + ((im73b1 != -10000) ? im73b1 : 0)" - -# For each pixel location, getting the count of valid pixels -count = sum([pyotb.where(ndvi == nodata, 0, 1) for ndvi in ndvis]) - -mean = summed / count # BandMath exp of this is very long: "(0 + ((im1b1 != -10000) ? im1b1 : 0) + ... + ((im73b1 != -10000) ? im73b1 : 0)) / (0 + ((im1b1 == -10000) ? 0 : 1) + ... + ((im73b1 == -10000) ? 0 : 1))" -mean.write('ndvi_annual_mean.tif') -``` - -Note that no actual computation is executed before the last line where the result is written to disk. - -### Process raw Pleiades data -This is a common case of Pleiades data preprocessing : optical calibration -> orthorectification -> pansharpening - -```python -import pyotb -srtm = '/media/data/raster/nasa/srtm_30m' -geoid = '/media/data/geoid/egm96.grd' +# RigidTransformResample, with input parameters as dict +resampled = pyotb.RigidTransformResample({ + "in": "https://myserver.ia/input.tif", # Note: no /vsicurl/ + "interpolator": "linear", + "transform.type.id.scaley": 0.5, + "transform.type.id.scalex": 0.5 +}) -pan = pyotb.OpticalCalibration('IMG_PHR1A_P_001/DIM_PHR1A_P_201509011347379_SEN_1791374101-001.XML', level='toa') -ms = pyotb.OpticalCalibration('IMG_PHR1A_MS_002/DIM_PHR1A_MS_201509011347379_SEN_1791374101-002.XML', level='toa') +# OpticalCalibration, with input parameters as args +calib = pyotb.OpticalCalibration(resampled) -pan_ortho = pyotb.OrthoRectification({'io.in': pan, 'elev.dem': srtm, 'elev.geoid': geoid}) -ms_ortho = pyotb.OrthoRectification({'io.in': ms, 'elev.dem': srtm, 'elev.geoid': geoid}) +# BandMath, with input parameters as kwargs +ndvi = pyotb.BandMath(calib, exp="ndvi(im1b1, im1b4)") -pxs = pyotb.BundleToPerfectSensor(inp=pan_ortho, inxs=ms_ortho, method='bayes', mode="default") +# Pythonic slicing +roi = ndvi[20:586, 9:572] -# Here we trigger every app in the pipeline and the process is blocked until result is written to disk -pxs.write('pxs_image.tif', pixel_type='uint16', filename_extension='?gdal:co:COMPRESS=DEFLATE&gdal:co:PREDICTOR=2') +# Pipeline execution. The actual computation happens here! +roi.write("output.tif", "float") ``` diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt index 3086078f8eb85fdd642ee996769ca9f3c7ebe090..d7f6094769170ed1c7bda85144efd50e2bff4983 100644 --- a/RELEASE_NOTES.txt +++ b/RELEASE_NOTES.txt @@ -1,3 +1,25 @@ +--------------------------------------------------------------------- +2.00 (Nov 23, 2023) - Changes since version 1.5.4 + +- Major refactoring (see troubleshooting/migration) +- Pythonic extended filenames (can use dict, etc) +- Easy access to image metadata +- CI improvements (tests, coverage, doc, etc) +- Documentation improvement +- Code format +- Allow OTB dotted parameters in kwargs +- Easy access to pixel coordinates +- Add function to transform x,y coordinates into row, col +- Native support of vsicurl inputs +- Fixes and enhancements in `summarize()` +- Fixes in `shape` +- Add typing to function defs to enhance documentation + +--------------------------------------------------------------------- +1.5.4 (Oct 01, 2022) - Changes since version 1.5.3 + +- Fix slicer wrong end of slicing + --------------------------------------------------------------------- 1.5.3 (Sep 29, 2022) - Changes since version 1.5.2 diff --git a/doc/MISC.md b/doc/MISC.md index 8da0eeace3eeb71cd0c20e92a88758fa5c072da2..6be696fe9f16702d33193efcf625b57c9423260e 100644 --- a/doc/MISC.md +++ b/doc/MISC.md @@ -1,9 +1,12 @@ ## Miscellaneous: Work with images with different footprints / resolutions -OrfeoToolBox provides a handy `Superimpose` application that enables the projection of an image into the geometry of another one. + +OrfeoToolBox provides a handy `Superimpose` application that enables the +projection of an image into the geometry of another one. In pyotb, a function has been created to handle more than 2 images. -Let's consider the case where we have 3 images with different resolutions and different footprints : +Let's consider the case where we have 3 images with different resolutions and +different footprints :  @@ -11,12 +14,15 @@ Let's consider the case where we have 3 images with different resolutions and di import pyotb # transforming filepaths to pyotb objects -s2_image, vhr_image, labels = pyotb.Input('image_10m.tif'), pyotb.Input('image_60cm.tif'), pyotb.Input('land_cover_2m.tif') +s2_image = pyotb.Input('image_10m.tif') +vhr_image = pyotb.Input('image_60cm.tif') +labels = pyotb.Input('land_cover_2m.tif') print(s2_image.shape) # (286, 195, 4) print(vhr_image.shape) # (2048, 2048, 3) print(labels.shape) # (1528, 1360, 1) ``` + Our goal is to obtain all images at the same footprint, same resolution and same shape. Let's consider we want the intersection of all footprints and the same resolution as `labels` image. @@ -25,11 +31,16 @@ Let's consider we want the intersection of all footprints and the same resolutio Here is the final result :  -The piece of code to achieve this : +The piece of code to achieve this : + ```python -s2_image, vhr_image, labels = pyotb.define_processing_area(s2_image, vhr_image, labels, window_rule='intersection', - pixel_size_rule='same_as_input', - reference_pixel_size_input=labels, interpolator='bco') +s2_image, vhr_image, labels = pyotb.define_processing_area( + s2_image, vhr_image, labels, + window_rule='intersection', + pixel_size_rule='same_as_input', + reference_pixel_size_input=labels, + interpolator='bco' +) print(s2_image.shape) # (657, 520, 4) print(vhr_image.shape) # (657, 520, 3) diff --git a/doc/comparison_otb.md b/doc/comparison_otb.md index 612eab6ffc46d02e91ca730a99c3059662e06dd5..0129cb2e866d9a990f7d7df3fc1f6a031de6e64b 100644 --- a/doc/comparison_otb.md +++ b/doc/comparison_otb.md @@ -1,132 +1,302 @@ ## Comparison between otbApplication and pyotb ### Single application execution -Using OTB, the code would be like : + +<table> +<tr> +<th> OTB </th> +<th> pyotb </th> +</tr> +<tr> +<td> + ```python -import otbApplication +import otbApplication as otb input_path = 'my_image.tif' -resampled = otbApplication.Registry.CreateApplication('RigidTransformResample') -resampled.SetParameterString('in', input_path) -resampled.SetParameterString('interpolator', 'linear') -resampled.SetParameterFloat('transform.type.id.scalex', 0.5) -resampled.SetParameterFloat('transform.type.id.scaley', 0.5) -resampled.SetParameterString('out', 'output.tif') -resampled.SetParameterOutputImagePixelType('out', otbApplication.ImagePixelType_uint16) -resampled.ExecuteAndWriteOutput() +app = otb.Registry.CreateApplication( + 'RigidTransformResample' +) +app.SetParameterString( + 'in', input_path +) +app.SetParameterString( + 'interpolator', 'linear' +) +app.SetParameterFloat( + 'transform.type.id.scalex', 0.5 +) +app.SetParameterFloat( + 'transform.type.id.scaley', 0.5 +) +app.SetParameterString( + 'out', 'output.tif' +) +app.SetParameterOutputImagePixelType( + 'out', otb.ImagePixelType_uint16 +) + +app.ExecuteAndWriteOutput() ``` -Using pyotb: +</td> +<td> + ```python import pyotb -resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'interpolator': 'linear', - 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5}) -resampled.write('output.tif', pixel_type='uint16') + +app = pyotb.RigidTransformResample({ + 'in': 'my_image.tif', + 'interpolator': 'linear', + 'transform.type.id.scaley': 0.5, + 'transform.type.id.scalex': 0.5 +}) + +app.write( + 'output.tif', + pixel_type='uint16' +) ``` +</td> +</tr> +</table> + ### In-memory connections -Using OTB : +<table> +<tr> +<th> OTB </th> +<th> pyotb </th> +</tr> +<tr> +<td> + ```python -import otbApplication - -resampled = otbApplication.Registry.CreateApplication('RigidTransformResample') -resampled.SetParameterString('in', 'my_image.tif') -resampled.SetParameterString('interpolator', 'linear') -resampled.SetParameterFloat('transform.type.id.scalex', 0.5) -resampled.SetParameterFloat('transform.type.id.scaley', 0.5) -resampled.Execute() - -calibrated = otbApplication.Registry.CreateApplication('OpticalCalibration') -calibrated.ConnectImage('in', resampled, 'out') -calibrated.SetParameterString('level', 'toa') -calibrated.Execute() - -dilated = otbApplication.Registry.CreateApplication('BinaryMorphologicalOperation') -dilated.ConnectImage('in', calibrated, 'out') -dilated.SetParameterString("filter", 'dilate') -dilated.SetParameterString("structype", 'ball') -dilated.SetParameterInt("xradius", 3) -dilated.SetParameterInt("yradius", 3) -dilated.SetParameterString('out', 'output.tif') -dilated.SetParameterOutputImagePixelType('out', otbApplication.ImagePixelType_uint16) -dilated.ExecuteAndWriteOutput() +import otbApplication as otb + +app1 = otb.Registry.CreateApplication( + 'RigidTransformResample' +) +app1.SetParameterString( + 'in', 'my_image.tif' +) +app1.SetParameterString( + 'interpolator', 'linear' +) +app1.SetParameterFloat( + 'transform.type.id.scalex', 0.5 +) +app1.SetParameterFloat( + 'transform.type.id.scaley', 0.5 +) +app1.Execute() + +app2 = otb.Registry.CreateApplication( + 'OpticalCalibration' +) +app2.ConnectImage('in', app1, 'out') +app2.SetParameterString('level', 'toa') +app2.Execute() + +app3 = otb.Registry.CreateApplication( + 'BinaryMorphologicalOperation' +) +app3.ConnectImage( + 'in', app2, 'out' +) +app3.SetParameterString( + 'filter', 'dilate' +) +app3.SetParameterString( + 'structype', 'ball' +) +app3.SetParameterInt( + 'xradius', 3 +) +app3.SetParameterInt( + 'yradius', 3 +) +app3.SetParameterString( + 'out', 'output.tif' +) +app3.SetParameterOutputImagePixelType( + 'out', otb.ImagePixelType_uint16 +) +app3.ExecuteAndWriteOutput() ``` -Using pyotb: +</td> +<td> + ```python import pyotb -resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'interpolator': 'linear', - 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5}) +app1 = pyotb.RigidTransformResample({ + 'in': 'my_image.tif', + 'interpolator': 'linear', + 'transform.type.id.scaley': 0.5, + 'transform.type.id.scalex': 0.5 +}) + +app2 = pyotb.OpticalCalibration({ + 'in': app1, + 'level': 'toa' +}) -calibrated = pyotb.OpticalCalibration({'in': resampled, 'level': 'toa'}) +app3 = pyotb.BinaryMorphologicalOperation({ + 'in': app2, + 'out': 'output.tif', + 'filter': 'dilate', + 'structype': 'ball', + 'xradius': 3, + 'yradius': 3 +}) -dilated = pyotb.BinaryMorphologicalOperation({'in': calibrated, 'out': 'output.tif', 'filter': 'dilate', - 'structype': 'ball', 'xradius': 3, 'yradius': 3}) -dilated.write('result.tif', pixel_type='uint16') +app3.write( + 'result.tif', + pixel_type='uint16' +) ``` +</td> +</tr> +</table> + ### Arithmetic operations -Every pyotb object supports arithmetic operations, such as addition, subtraction, comparison... -Consider an example where we want to perform the arithmetic operation `image1 * image2 - 2*image3` -Using OTB, the following code works for 3-bands images : +Every pyotb object supports arithmetic operations, such as addition, +subtraction, comparison... +Consider an example where we want to perform the arithmetic operation +`image1 * image2 - 2*image3`. + +<table> +<tr> +<th> OTB </th> +<th> pyotb </th> +</tr> +<tr> +<td> + ```python -import otbApplication +import otbApplication as otb -bmx = otbApplication.Registry.CreateApplication('BandMathX') -bmx.SetParameterStringList('il', ['image1.tif', 'image2.tif', 'image3.tif']) # all images are 3-bands -exp = 'im1b1*im2b1 - 2*im3b1; im1b2*im2b2 - 2*im3b2; im1b3*im2b3 - 2*im3b3' +bmx = otb.Registry.CreateApplication( + 'BandMathX' +) +bmx.SetParameterStringList( + 'il', + ['im1.tif', 'im2.tif', 'im3.tif'] +) +exp = ('im1b1*im2b1-2*im3b1; ' + 'im1b2*im2b2-2*im3b2; ' + 'im1b3*im2b3-2*im3b3') bmx.SetParameterString('exp', exp) -bmx.SetParameterString('out', 'output.tif') -bmx.SetParameterOutputImagePixelType('out', otbApplication.ImagePixelType_uint8) +bmx.SetParameterString( + 'out', + 'output.tif' +) +bmx.SetParameterOutputImagePixelType( + 'out', + otb.ImagePixelType_uint8 +) bmx.ExecuteAndWriteOutput() ``` -With pyotb, the following works with images of any number of bands : +Note: code limited to 3-bands images. + +</td> +<td> + ```python import pyotb -# transforming filepaths to pyotb objects -input1, input2, input3 = pyotb.Input('image1.tif'), pyotb.Input('image2.tif') , pyotb.Input('image3.tif') +# filepaths --> pyotb objects +in1 = pyotb.Input('im1.tif') +in2 = pyotb.Input('im2.tif') +in3 = pyotb.Input('im3.tif') -res = input1 * input2 - 2 * input2 -res.write('output.tif', pixel_type='uint8') +res = in1 * in2 - 2 * in3 +res.write( + 'output.tif', + pixel_type='uint8' +) ``` +Note: works with any number of bands. + +</td> +</tr> +</table> + ### Slicing -Using OTB, for selection bands or ROI, the code looks like: +<table> +<tr> +<th> OTB </th> +<th> pyotb </th> +</tr> +<tr> +<td> + + ```python -import otbApplication - -# selecting first 3 bands -extracted = otbApplication.Registry.CreateApplication('ExtractROI') -extracted.SetParameterString('in', 'my_image.tif') -extracted.SetParameterStringList('cl', ['Channel1', 'Channel2', 'Channel3']) -extracted.Execute() - -# selecting 1000x1000 subset -extracted = otbApplication.Registry.CreateApplication('ExtractROI') -extracted.SetParameterString('in', 'my_image.tif') -extracted.SetParameterString('mode', 'extent') -extracted.SetParameterString('mode.extent.unit', 'pxl') -extracted.SetParameterFloat('mode.extent.ulx', 0) -extracted.SetParameterFloat('mode.extent.uly', 0) -extracted.SetParameterFloat('mode.extent.lrx', 999) -extracted.SetParameterFloat('mode.extent.lry', 999) -extracted.Execute() +import otbApplication as otb + +# first 3 channels +app = otb.Registry.CreateApplication( + 'ExtractROI' +) +app.SetParameterString( + 'in', 'my_image.tif' +) +app.SetParameterStringList( + 'cl', + ['Channel1', 'Channel2', 'Channel3'] +) +app.Execute() + +# 1000x1000 roi +app = otb.Registry.CreateApplication( + 'ExtractROI' +) +app.SetParameterString( + 'in', 'my_image.tif' +) +app.SetParameterString( + 'mode', 'extent' +) +app.SetParameterString( + 'mode.extent.unit', 'pxl' +) +app.SetParameterFloat( + 'mode.extent.ulx', 0 +) +app.SetParameterFloat( + 'mode.extent.uly', 0 +) +app.SetParameterFloat( + 'mode.extent.lrx', 999 +) +app.SetParameterFloat( + 'mode.extent.lry', 999 +) +app.Execute() ``` -Instead, using pyotb: +</td> +<td> ```python import pyotb -# transforming filepath to pyotb object +# filepath --> pyotb object inp = pyotb.Input('my_image.tif') -extracted = inp[:, :, :3] # selecting first 3 bands -extracted = inp[:1000, :1000] # selecting 1000x1000 subset -``` \ No newline at end of file +extracted = inp[:, :, :3] # Bands 1,2,3 +extracted = inp[:1000, :1000] # ROI +``` + +</td> +</tr> +</table> \ No newline at end of file diff --git a/doc/examples/nodata_mean.md b/doc/examples/nodata_mean.md index 9ec49002294cf6f66950ce989ac3a447070c32dd..288be6e8fb48315ff946d0e9b5505fb49e9c4a58 100644 --- a/doc/examples/nodata_mean.md +++ b/doc/examples/nodata_mean.md @@ -1,7 +1,9 @@ ### Compute the mean of several rasters, taking into account NoData -Let's consider we have at disposal 73 NDVI rasters for a year, where clouds have been masked with NoData (nodata value of -10 000 for example). +Let's consider we have at disposal 73 NDVI rasters for a year, where clouds +have been masked with NoData (nodata value of -10 000 for example). -Goal: compute the mean across time (keeping the spatial dimension) of the NDVIs, excluding cloudy pixels. Piece of code to achieve that: +Goal: compute the mean across time (keeping the spatial dimension) of the +NDVIs, excluding cloudy pixels. Piece of code to achieve that: ```python import pyotb @@ -13,14 +15,21 @@ ndvis = [pyotb.Input(path) for path in ndvi_paths] summed = sum([pyotb.where(ndvi != nodata, ndvi, 0) for ndvi in ndvis]) # Printing the generated BandMath expression -print(summed.exp) # this returns a very long exp: "0 + ((im1b1 != -10000) ? im1b1 : 0) + ((im2b1 != -10000) ? im2b1 : 0) + ... + ((im73b1 != -10000) ? im73b1 : 0)" +print(summed.exp) +# this returns a very long exp: +# "0 + ((im1b1 != -10000) ? im1b1 : 0) + ((im2b1 != -10000) ? im2b1 : 0) + ... +# ... + ((im73b1 != -10000) ? im73b1 : 0)" # For each pixel location, getting the count of valid pixels count = sum([pyotb.where(ndvi == nodata, 0, 1) for ndvi in ndvis]) -mean = summed / count # BandMath exp of this is very long: "(0 + ((im1b1 != -10000) ? im1b1 : 0) + ... + ((im73b1 != -10000) ? im73b1 : 0)) / (0 + ((im1b1 == -10000) ? 0 : 1) + ... + ((im73b1 == -10000) ? 0 : 1))" +mean = summed / count +# BandMath exp of this is very long: +# "(0 + ((im1b1 != -10000) ? im1b1 : 0) + ... +# + ((im73b1 != -10000) ? im73b1 : 0)) / (0 + ((im1b1 == -10000) ? 0 : 1) + ... +# + ((im73b1 == -10000) ? 0 : 1))" mean.write('ndvi_annual_mean.tif') ``` -Note that no actual computation is executed before the last line where the result is written to disk. - +Note that no actual computation is executed before the last line where the +result is written to disk. diff --git a/doc/examples/pleiades.md b/doc/examples/pleiades.md index 32d0447bba7361013d19d17bfe6f2a37074b7a7c..120a5542d03f519fcb54ff583c484f9212597aa1 100644 --- a/doc/examples/pleiades.md +++ b/doc/examples/pleiades.md @@ -1,20 +1,41 @@ ### Process raw Pleiades data -This is a common case of Pleiades data preprocessing : optical calibration -> orthorectification -> pansharpening +This is a common case of Pleiades data preprocessing : +*optical calibration -> orthorectification -> pansharpening* ```python import pyotb srtm = '/media/data/raster/nasa/srtm_30m' geoid = '/media/data/geoid/egm96.grd' -pan = pyotb.OpticalCalibration('IMG_PHR1A_P_001/DIM_PHR1A_P_201509011347379_SEN_1791374101-001.XML', level='toa') -ms = pyotb.OpticalCalibration('IMG_PHR1A_MS_002/DIM_PHR1A_MS_201509011347379_SEN_1791374101-002.XML', level='toa') +pan = pyotb.OpticalCalibration( + 'IMG_PHR1A_P_001/DIM_PHR1A_P_201509011347379_SEN_1791374101-001.XML', + level='toa' +) +ms = pyotb.OpticalCalibration( + 'IMG_PHR1A_MS_002/DIM_PHR1A_MS_201509011347379_SEN_1791374101-002.XML', + level='toa' +) -pan_ortho = pyotb.OrthoRectification({'io.in': pan, 'elev.dem': srtm, 'elev.geoid': geoid}) -ms_ortho = pyotb.OrthoRectification({'io.in': ms, 'elev.dem': srtm, 'elev.geoid': geoid}) +pan_ortho = pyotb.OrthoRectification({ + 'io.in': pan, + 'elev.dem': srtm, + 'elev.geoid': geoid +}) +ms_ortho = pyotb.OrthoRectification({ + 'io.in': ms, + 'elev.dem': srtm, + 'elev.geoid': geoid +}) -pxs = pyotb.BundleToPerfectSensor(inp=pan_ortho, inxs=ms_ortho, method='bayes', mode='default') +pxs = pyotb.BundleToPerfectSensor( + inp=pan_ortho, + inxs=ms_ortho, + method='bayes', + mode='default' +) exfn = '?gdal:co:COMPRESS=DEFLATE&gdal:co:PREDICTOR=2&gdal:co:BIGTIFF=YES' -# Here we trigger every app in the pipeline and the process is blocked until result is written to disk -pxs.write('pxs_image.tif', pixel_type='uint16', filename_extension=exfn) +# Here we trigger every app in the pipeline and the process is blocked until +# result is written to disk +pxs.write('pxs_image.tif', pixel_type='uint16', ext_fname=exfn) ``` diff --git a/doc/extra.css b/doc/extra.css new file mode 100644 index 0000000000000000000000000000000000000000..ec4d4bbd55f0bdf33df78446fa2437cfa8a69def --- /dev/null +++ b/doc/extra.css @@ -0,0 +1,11 @@ +.rst-content div[class^=highlight] { + border: 0px; +} + +.rst-content div[class^=highlight] pre { + padding: 0px; +} + +.rst-content pre code { + background: #eeffcc; +} diff --git a/doc/features.md b/doc/features.md index 7bc78ce1e318cbb3a9151dda019f9369d0b4631d..3a55c038f5218a226d4c00481ff262b000f3548d 100644 --- a/doc/features.md +++ b/doc/features.md @@ -1,8 +1,12 @@ ## Arithmetic operations -Every pyotb object supports arithmetic operations, such as addition, subtraction, comparison... -Consider an example where we want to compute a vegeteation mask from NDVI, i.e. the arithmetic operation `(nir - red) / (nir + red) > 0.3` + +Every pyotb object supports arithmetic operations, such as addition, +subtraction, comparison... +Consider an example where we want to compute a vegeteation mask from NDVI, +i.e. the arithmetic operation `(nir - red) / (nir + red) > 0.3` With pyotb, one can simply do : + ```python import pyotb @@ -10,11 +14,14 @@ import pyotb nir, red = pyotb.Input('nir.tif'), pyotb.Input('red.tif') res = (nir - red) / (nir + red) > 0.3 -print(res.exp) # prints the BandMath expression: "((im1b1 - im2b1) / (im1b1 + im2b1)) > 0.3 ? 1 : 0" +# Prints the BandMath expression: +# "((im1b1 - im2b1) / (im1b1 + im2b1)) > 0.3 ? 1 : 0" +print(res.exp) res.write('vegetation_mask.tif', pixel_type='uint8') ``` ## Slicing + pyotb objects support slicing in a Python fashion : ```python @@ -25,20 +32,154 @@ inp = pyotb.Input('my_image.tif') inp[:, :, :3] # selecting first 3 bands inp[:, :, [0, 1, 4]] # selecting bands 1, 2 & 5 -inp[:1000, :1000] # selecting 1000x1000 subset, same as inp[:1000, :1000, :] +inp[:, :, 1:-1] # removing first and last band +inp[:, :, ::2] # selecting one band every 2 bands +inp[:100, :100] # selecting 100x100 subset, same as inp[:100, :100, :] inp[:100, :100].write('my_image_roi.tif') # write cropped image to disk ``` -## Shape attributes -You can access the shape of any in-memory pyotb object. +## Retrieving a pixel location in image coordinates + +One can retrieve a pixel location in image coordinates (i.e. row and column +indices) using `get_rowcol_from_xy()`: ```python -import pyotb +inp.get_rowcol_from_xy(760086.0, 6948092.0) # (333, 5) +``` -# transforming filepath to pyotb object -inp = pyotb.Input('my_image.tif') +## Reading a pixel value + +One can read a pixel value of a pyotb object using brackets, as if it was a +common array. Returned is a list of pixel values for each band: + +```python +inp[10, 10] # [217, 202, 182, 255] +``` + +!!! warning + + Accessing multiple pixels values if not computationally efficient. Please + use this with moderation, or consider numpy or pyotb applications to + process efficiently blocks of pixels. + +## Attributes + +### Shape + +The shape of pyotb objects can be retrieved using `shape`. + +```python print(inp[:1000, :500].shape) # (1000, 500, 4) ``` +### Pixel type + +The pixel type of pyotb objects can be retrieved using `dtype`. + +```python +inp.dtype # e.g. 'uint8' +``` + +!!! note + + The `dtype` returns a `str` corresponding to values accepted by the + `pixel_type` of `write()` + +### Transform + +The transform, as defined in GDAL, can be retrieved with the `transform` +attribute: + +```python +inp.transform # (6.0, 0.0, 760056.0, 0.0, -6.0, 6946092.0) +``` + +### Metadata + +Images metadata can be retrieved with the `metadata` attribute: + +```python +print(inp.metadata) +``` + +Gives: + +``` +{ + 'DataType': 1.0, + 'DriverLongName': 'GeoTIFF', + 'DriverShortName': 'GTiff', + 'GeoTransform': (760056.0, 6.0, 0.0, 6946092.0, 0.0, -6.0), + 'LowerLeftCorner': (760056.0, 6944268.0), + 'LowerRightCorner': (761562.0, 6944268.0), + 'AREA_OR_POINT': 'Area', + 'TIFFTAG_SOFTWARE': 'CSinG - 13 SEPTEMBRE 2012', + 'ProjectionRef': 'PROJCS["RGF93 v1 / Lambert-93",\n...', + 'ResolutionFactor': 0, + 'SubDatasetIndex': 0, + 'UpperLeftCorner': (760056.0, 6946092.0), + 'UpperRightCorner': (761562.0, 6946092.0), + 'TileHintX': 251.0, + 'TileHintY': 8.0 +} +``` + +## Information + +The information fetched by the `ReadImageInfo` OTB application is available +through `get_info()`: + +```python +print(inp.get_info()) +``` + +Gives: + +```json lines +{ + 'indexx': 0, + 'indexy': 0, + 'sizex': 251, + 'sizey': 304, + 'spacingx': 6.0, + 'spacingy': -6.0, + 'originx': 760059.0, + 'originy': 6946089.0, + 'estimatedgroundspacingx': 5.978403091430664, + 'estimatedgroundspacingy': 5.996793270111084, + 'numberbands': 4, + 'datatype': 'unsigned_char', + 'ullat': 0.0, + 'ullon': 0.0, + 'urlat': 0.0, + 'urlon': 0.0, + 'lrlat': 0.0, + 'lrlon': 0.0, + 'lllat': 0.0, + 'lllon': 0.0, + 'rgb.r': 0, + 'rgb.g': 1, + 'rgb.b': 2, + 'projectionref': 'PROJCS["RGF93 v1 ..."EPSG","2154"]]', + 'gcp.count': 0 +} +``` + +## Statistics + +Image statistics can be computed on-the-fly using `get_statistics()`: + +```python +print(inp.get_statistics()) +``` +Gives: +```json lines +{ + 'out.mean': [79.5505, 109.225, 115.456, 249.349], + 'out.min': [33, 64, 91, 47], + 'out.max': [255, 255, 230, 255], + 'out.std': [51.0754, 35.3152, 23.4514, 20.3827] +} +``` \ No newline at end of file diff --git a/doc/functions.md b/doc/functions.md index 070ba0be35431fb2fe3191a291f85c5251edc09d..1e82c58d0c125fb68a0dac2f1b5ef742760cae4c 100644 --- a/doc/functions.md +++ b/doc/functions.md @@ -1,26 +1,47 @@ -Some functions have been written, entirely based on OTB, to mimic the behavior of some well-known numpy functions. +Some functions have been written, entirely based on OTB, to mimic the behavior +of some well-known numpy functions. + ## pyotb.where + Equivalent of `numpy.where`. -It is the equivalent of the muparser syntax `condition ? x : y` that can be used in OTB's BandMath. +It is the equivalent of the muparser syntax `condition ? x : y` that can be +used in OTB's BandMath. ```python import pyotb # transforming filepaths to pyotb objects -labels, image1, image2 = pyotb.Input('labels.tif'), pyotb.Input('image1.tif') , pyotb.Input('image2.tif') +labels = pyotb.Input('labels.tif') +image1 = pyotb.Input('image1.tif') +image2 = pyotb.Input('image2.tif') # If labels = 1, returns image1. Else, returns image2 -res = pyotb.where(labels == 1, image1, image2) # this would also work: pyotb.where(labels == 1, 'image1.tif', 'image2.tif') +res = pyotb.where(labels == 1, image1, image2) +# this would also work: `pyotb.where(labels == 1, 'image1.tif', 'image2.tif')` -# A more complex example -# If labels = 1, returns image1. If labels = 2, returns image2. If labels = 3, returns 3. Else 0 -res = pyotb.where(labels == 1, image1, - pyotb.where(labels == 2, image2, - pyotb.where(labels == 3, 3, 0))) +# A more complex example: +# - If labels = 1 --> returns image1, +# - If labels = 2 --> returns image2, +# - If labels = 3 --> returns 3.0, +# - Else, returns 0.0 +res = pyotb.where( + labels == 1, + image1, + pyotb.where( + labels == 2, + image2, + pyotb.where( + labels == 3, + 3.0, + 0.0 + ) + ) +) ``` ## pyotb.clip + Equivalent of `numpy.clip`. Clip (limit) the values in a raster to a range. ```python @@ -30,18 +51,20 @@ res = pyotb.clip('my_image.tif', 0, 255) # clips the values between 0 and 255 ``` ## pyotb.all + Equivalent of `numpy.all`. -For only one image, this function checks that all bands of the image are True (i.e. !=0) and outputs -a singleband boolean raster. -For several images, this function checks that all images are True (i.e. !=0) and outputs -a boolean raster, with as many bands as the inputs. +For only one image, this function checks that all bands of the image are True +(i.e. !=0) and outputs a single band boolean raster. +For several images, this function checks that all images are True (i.e. !=0) +and outputs a boolean raster, with as many bands as the inputs. ## pyotb.any + Equivalent of `numpy.any`. -For only one image, this function checks that at least one band of the image is True (i.e. !=0) and outputs -a singleband boolean raster. -For several images, this function checks that at least one of the images is True (i.e. !=0) and outputs -a boolean raster, with as many bands as the inputs. \ No newline at end of file +For only one image, this function checks that at least one band of the image +is True (i.e. !=0) and outputs a single band boolean raster. +For several images, this function checks that at least one of the images is +True (i.e. !=0) and outputs a boolean raster, with as many bands as the inputs. \ No newline at end of file diff --git a/doc/index.md b/doc/index.md index fa29918aa9d08c9b401e5140f56078bcd34803b9..4ecfaf4462efc96263c3e39b4de2237bc6968d16 100644 --- a/doc/index.md +++ b/doc/index.md @@ -1,26 +1,43 @@ -# Pyotb: Orfeo Toolbox for Python +# pyotb: Orfeo Toolbox for Python -pyotb is a Python extension of Orfeo Toolbox. It has been built on top of the existing Python API of OTB, in order +pyotb is a Python extension of Orfeo Toolbox. It has been built on top of the +existing Python API of OTB, in order to make OTB more Python friendly. # Table of Contents ## Get started + - [Installation](installation.md) -- [How to use pyotb](quickstart.md) +- [Quick start](quickstart.md) - [Useful features](features.md) -- [Functions](features.md) +- [Functions](functions.md) - [Interaction with Python libraries (numpy, rasterio, tensorflow)](interaction.md) ## Examples + - [Pleiades data processing](examples/pleiades.md) - [Computing the mean of several rasters with NoData](examples/nodata_mean.md) ## Advanced use + - [Comparison between pyotb and OTB native library](comparison_otb.md) +- [Summarize applications](summarize.md) - [OTB versions](otb_versions.md) - [Managing loggers](managing_loggers.md) - [Troubleshooting & limitations](troubleshooting.md) - ## API + +- See the API reference. If you have any doubts or questions, feel free to ask +on github or gitlab! + +## Contribute + +Contributions are welcome ! +Open a PR/MR, or file an issue if you spot a bug or have any suggestion: + +- [Github](https://github.com/orfeotoolbox/pyotb) +- [Orfeo ToolBox GitLab instance](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb). + +Thank you! \ No newline at end of file diff --git a/doc/installation.md b/doc/installation.md index dd7b729f3281825483d8937bf5022bcf7b9e0554..404d7bd7d228ec6a0e850b7dcd8ba214f57b69c3 100644 --- a/doc/installation.md +++ b/doc/installation.md @@ -1,11 +1,28 @@ +## Prerequisites -## Requirements -- Python>=3.5 and NumPy -- Orfeo ToolBox python API (instructions available [on the official website](https://www.orfeo-toolbox.org/CookBook/Installation.html)) +Requirements: + +- Python >= 3.7 and NumPy +- Orfeo ToolBox binaries (follow these + [instructions](https://www.orfeo-toolbox.org/CookBook/Installation.html)) +- Orfeo ToolBox python binding (follow these + [instructions](https://www.orfeo-toolbox.org/CookBook/Installation.html#python-bindings)) + +## Install with pip -## Installation ```bash pip install pyotb --upgrade ``` -For Python>=3.6, latest version available is pyotb 1.5.0. For Python 3.5, latest version available is pyotb 1.2.2 +For development, use the following: + +```bash +git clone https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb +cd pyotb +pip install -e ".[dev]" +``` + +## Old versions + +If you need compatibility with python3.6, install `pyotb<2.0` and for + python3.5 use `pyotb==1.2.2`. diff --git a/doc/interaction.md b/doc/interaction.md index d44ca68d240ed7561773f4cf244d24632b4fab2f..dcb87290936f6f60b65e7b66e4def90081e8c02d 100644 --- a/doc/interaction.md +++ b/doc/interaction.md @@ -1,17 +1,24 @@ -## Export to Numpy +## Numpy + +### Export to numpy arrays pyotb objects can be exported to numpy array. + ```python import pyotb import numpy as np -calibrated = pyotb.OpticalCalibration('image.tif', level='toa') # this is a pyotb object -arr = np.asarray(calibrated) # same as calibrated.to_numpy() +# The following is a pyotb object +calibrated = pyotb.OpticalCalibration('image.tif', level='toa') -``` +# The following is a numpy array +arr = np.asarray(calibrated) +# Note that the following is equivalent: +arr = calibrated.to_numpy() +``` -## Interaction with Numpy +### Interact with numpy functions pyotb objects can be transparently used in numpy functions. @@ -21,24 +28,30 @@ For example: import pyotb import numpy as np -inp = pyotb.Input('image.tif') # this is a pyotb object +# The following is a pyotb object +inp = pyotb.Input('image.tif') -# Creating a numpy array of noise -white_noise = np.random.normal(0, 50, size=inp.shape) # this is a numpy object +# Creating a numpy array of noise. The following is a numpy object +white_noise = np.random.normal(0, 50, size=inp.shape) # Adding the noise to the image -noisy_image = inp + white_noise # magic: this is a pyotb object that has the same georeference as input. - # `np.add(inp, white_noise)` would have worked the same +noisy_image = inp + white_noise +# Magically, this is a pyotb object that has the same geo-reference as `inp`. +# Note the `np.add(inp, white_noise)` would have worked the same + +# Finally we can write the result like any pyotb object noisy_image.write('image_plus_noise.tif') ``` -Limitations : -- The whole image is loaded into memory -- The georeference can not be modified. Thus, numpy operations can not change the image or pixel size +!!! warning + - The whole image is loaded into memory + - The georeference can not be modified. Thus, numpy operations can not + change the image or pixel size -## Export to rasterio -pyotb objects can also be exported in a format that is usable by rasterio. +## Rasterio + +pyotb objects can also be exported in a format usable by rasterio. For example: @@ -48,32 +61,42 @@ import rasterio from scipy import ndimage # Pansharpening + NDVI + creating bare soils mask -pxs = pyotb.BundleToPerfectSensor(inp='panchromatic.tif', inxs='multispectral.tif') -ndvi = pyotb.RadiometricIndices({'in': pxs, 'channels.red': 3, 'channels.nir': 4, 'list': 'Vegetation:NDVI'}) +pxs = pyotb.BundleToPerfectSensor( + inp='panchromatic.tif', + inxs='multispectral.tif' +) +ndvi = pyotb.RadiometricIndices({ + 'in': pxs, + 'channels.red': 3, + 'channels.nir': 4, + 'list': 'Vegetation:NDVI' +}) bare_soil_mask = (ndvi < 0.3) # Exporting the result as array & profile usable by rasterio mask_array, profile = bare_soil_mask.to_rasterio() -# Doing something in Python that is not possible with OTB, e.g. gathering the contiguous groups of pixels -# with an integer index +# Doing something in Python that is not possible with OTB, e.g. gathering +# the contiguous groups of pixels with an integer index labeled_mask_array, nb_groups = ndimage.label(mask_array) # Writing the result to disk with rasterio.open('labeled_bare_soil.tif', 'w', **profile) as f: f.write(labeled_mask_array) - ``` -This way of exporting pyotb objects is more flexible that exporting to numpy, as the user gets the `profile` dictionary. -If the georeference or pixel size is modified, the user can update the `profile` accordingly. - -## Interaction with Tensorflow +This way of exporting pyotb objects is more flexible that exporting to numpy, +as the user gets the `profile` dictionary. +If the georeference or pixel size is modified, the user can update the +`profile` accordingly. -We saw that numpy operations had some limitations. To bypass those limitations, it is possible to use some Tensorflow operations on pyotb objects. +## Tensorflow +We saw that numpy operations had some limitations. To bypass those +limitations, it is possible to use some Tensorflow operations on pyotb objects. -You need a working installation of OTBTF >=3.0 for this and then the code is like this: +You need a working installation of OTBTF >=3.0 for this and then the code is +like this: ```python import pyotb @@ -84,28 +107,37 @@ def scalar_product(x1, x2): return tf.reduce_sum(tf.multiply(x1, x2), axis=-1) # Compute the scalar product -res = pyotb.run_tf_function(scalar_product)('image1.tif', 'image2.tif') # magic: this is a pyotb object +res = pyotb.run_tf_function(scalar_product)('image1.tif', 'image2.tif') + +# Magically, `res` is a pyotb object res.write('scalar_product.tif') ``` -For some easy syntax, one can use `pyotb.run_tf_function` as a function decorator, such as: +For some easy syntax, one can use `pyotb.run_tf_function` as a function +decorator, such as: + ```python import pyotb -@pyotb.run_tf_function # The decorator enables the use of pyotb objects as inputs/output of the function +# The `pyotb.run_tf_function` decorator enables the use of pyotb objects as +# inputs/output of the function +@pyotb.run_tf_function def scalar_product(x1, x2): import tensorflow as tf return tf.reduce_sum(tf.multiply(x1, x2), axis=-1) -res = scalar_product('image1.tif', 'image2.tif') # magic: this is a pyotb object +res = scalar_product('image1.tif', 'image2.tif') +# Magically, `res` is a pyotb object ``` Advantages : -- The process supports streaming, hence the whole image is **not** loaded into memory +- The process supports streaming, hence the whole image is **not** loaded into +memory - Can be integrated in OTB pipelines -Limitations : +!!! warning -- It is not possible to use the tensorflow python API inside a script where OTBTF is used because of compilation issues -between Tensorflow and OTBTF, i.e. `import tensorflow` doesn't work in a script where OTBTF apps have been initialized + Due to compilation issues in OTBTF before version 4.0.0, tensorflow and + pyotb can't be imported in the same python code. This problem has been + fixed in OTBTF 4.0.0. diff --git a/doc/managing_loggers.md b/doc/managing_loggers.md index f329d5ed6e06c27633b58fd1005b8e04a5959f56..7220af482d781ad00fb62a69557def803b8d2d71 100644 --- a/doc/managing_loggers.md +++ b/doc/managing_loggers.md @@ -1,25 +1,32 @@ ## Managing loggers -Several environment variables are used in order to adjust logger level and behaviour. It should be set before importing pyotb. +Several environment variables are used in order to adjust logger level and +behaviour. It should be set before importing pyotb. - `OTB_LOGGER_LEVEL` : used to set the default OTB logger level. -- `PYOTB_LOGGER_LEVEL` : used to set the pyotb logger level. if not set, `OTB_LOGGER_LEVEL` will be used. +- `PYOTB_LOGGER_LEVEL` : used to set the pyotb logger level. if not set, +- `OTB_LOGGER_LEVEL` will be used. If none of those two variables is set, the logger level will be set to 'INFO'. Available levels are : DEBUG, INFO, WARNING, ERROR, CRITICAL -You may also change the logger level after import (for pyotb only) with the function `set_logger_level`. +You may also change the logger level after import (for pyotb only) with the +function `set_logger_level`. + ```python import pyotb pyotb.set_logger_level('DEBUG') ``` -Bonus : in some cases, yo may want to silence the GDAL driver logger (for example you will see a lot of errors when reading GML files with OGR). -One useful trick is to redirect these logs to a file. This can be done using the variable `CPL_LOG`. +Bonus : in some cases, you may want to silence the GDAL driver logger (for +example you will see a lot of errors when reading GML files with OGR). +One useful trick is to redirect these logs to a file. This can be done using +the variable `CPL_LOG`. ## Named applications in logs It is possible to change an app name in order to track it easily in the logs : + ```python import os os.environ['PYOTB_LOGGER_LEVEL'] = 'DEBUG' @@ -29,8 +36,9 @@ bm = pyotb.BandMath(['image.tif'], exp='im1b1 * 100') bm.name = 'CustomBandMathApp' bm.execute() ``` + ```text 2022-06-14 14:22:38 (DEBUG) [pyOTB] CustomBandMathApp: run execute() with parameters={'exp': 'im1b1 * 100', 'il': ['/home/vidlb/Téléchargements/test_4b.tif']} 2022-06-14 14:22:38 (INFO) BandMath: Image #1 has 4 components 2022-06-14 14:22:38 (DEBUG) [pyOTB] CustomBandMathApp: execution succeeded -``` \ No newline at end of file +``` diff --git a/doc/otb_versions.md b/doc/otb_versions.md index 6b06c28135d25a02bd792069f4d417d4ec64babd..aa8889d7a3b21edfe30f142e88d32412678b69fa 100644 --- a/doc/otb_versions.md +++ b/doc/otb_versions.md @@ -1,21 +1,27 @@ ## System with multiple OTB versions -If you want to quickly switch between OTB versions, or override the default system version, you may use the `OTB_ROOT` env variable : +If you want to quickly switch between OTB versions, or override the default +system version, you may use the `OTB_ROOT` env variable : + ```python import os # This is equivalent to "[set/export] OTB_ROOT=/opt/otb" before launching python os.environ['OTB_ROOT'] = '/opt/otb' import pyotb ``` + ```text 2022-06-14 13:59:03 (INFO) [pyOTB] Preparing environment for OTB in /opt/otb 2022-06-14 13:59:04 (INFO) [pyOTB] Successfully loaded 126 OTB applications ``` -If you try to import pyotb without having set environment, it will try to find any OTB version installed on your system: +If you try to import pyotb without having set environment, it will try to find +any OTB version installed on your system: + ```python import pyotb ``` + ```text 2022-06-14 13:55:41 (INFO) [pyOTB] Failed to import OTB. Searching for it... 2022-06-14 13:55:41 (INFO) [pyOTB] Found /opt/otb/lib/otb/ @@ -24,21 +30,30 @@ import pyotb 2022-06-14 13:55:43 (INFO) [pyOTB] Preparing environment for OTB in /home/otbuser/Applications/OTB-8.0.1-Linux64 2022-06-14 13:55:44 (INFO) [pyOTB] Successfully loaded 117 OTB applications ``` + Here is the path precedence for this automatic env configuration : + ```text OTB_ROOT env variable > python bindings directory OR search for releases installations : HOME OR (for linux) : /opt/otbtf > /opt/otb > /usr/local > /usr OR (for windows) : C:/Program Files ``` -N.B. : in case `otbApplication` is found in `PYTHONPATH` (and if `OTB_ROOT` was not set), the OTB which the python API is linked to will be used. + +!!! Note + + When `otbApplication` is found in `PYTHONPATH` (and `OTB_ROOT` not set), + the OTB installation where the python API is linked, will be used. ## Fresh OTB installation -If you've just installed OTB binaries in a Linux environment, you may encounter an error at first import, pyotb will help you fix it : +If you've just installed OTB binaries in a Linux environment, you may +encounter an error at first import, pyotb will help you fix it : + ```python import pyotb ``` + ```text 2022-06-14 14:00:34 (INFO) [pyOTB] Preparing environment for OTB in /home/otbuser/Applications/OTB-8.0.1-Linux64 2022-07-07 16:56:04 (CRITICAL) [pyOTB] An error occurred while importing OTB Python API diff --git a/doc/quickstart.md b/doc/quickstart.md index 294d5123263ebfcd08bce4bb569e7d3756924b4f..fb15188d8f06ddca795b361b87485e9ef74f5f03 100644 --- a/doc/quickstart.md +++ b/doc/quickstart.md @@ -1,73 +1,206 @@ ## Quickstart: running an OTB application with pyotb -pyotb has been written so that it is more convenient to run an application in Python. + +pyotb has been written so that it is more convenient to run an application in +Python. You can pass the parameters of an application as a dictionary : ```python import pyotb -resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'transform.type.id.scaley': 0.5, - 'interpolator': 'linear', 'transform.type.id.scalex': 0.5}) +resampled = pyotb.RigidTransformResample({ + 'in': 'my_image.tif', + 'transform.type.id.scaley': 0.5, + 'interpolator': 'linear', + 'transform.type.id.scalex': 0.5 +}) ``` -Note that pyotb has a 'lazy' evaluation: it only performs operation when it is needed, i.e. results are written to disk. -Thus, the previous line doesn't trigger the application. +For now, `resampled` has not been executed. Indeed, pyotb has a 'lazy' +evaluation: applications are executed only when required. Generally, like in +this example, executions happen to write output images to disk. -To actually trigger the application execution, you need to write the result to disk: +To actually trigger the application execution, `write()` has to be called: ```python resampled.write('output.tif') # this is when the application actually runs ``` ### Using Python keyword arguments -It is also possible to use the Python keyword arguments notation for passing the parameters: + +One can use the Python keyword arguments notation for passing that parameters: + ```python output = pyotb.SuperImpose(inr='reference_image.tif', inm='image.tif') ``` -is equivalent to: + +Which is equivalent to: + ```python output = pyotb.SuperImpose({'inr': 'reference_image.tif', 'inm': 'image.tif'}) ``` -Limitations : for this notation, python doesn't accept the parameter `in` or any parameter that contains a `.`. E.g., it is not possible to use `pyotb.RigidTransformResample(in=input_path...)` or `pyotb.VectorDataExtractROI(io.vd=vector_path...)`. +!!! warning + + For this notation, python doesn't accept the parameter `in` or any + parameter that contains a dots (e.g. `io.in`). For `in` or other main + input parameters of an OTB application, you may simply pass the value as + first argument, pyotb will guess the parameter name. For parameters that + contains dots, you can either use a dictionary, or replace dots (`.`) + with underscores (`_`). + + Let's take the example of the `OrthoRectification` application of OTB, + with the input image parameter named `io.in`: + + Option #1, keyword-arg-free: + + ```python + ortho = pyotb.OrthoRectification('my_image.tif') + ``` + + Option #2, replacing dots with underscores in parameter name: + + ```python + ortho = pyotb.OrthoRectification(io_in='my_image.tif') + ``` ## In-memory connections -The big asset of pyotb is the ease of in-memory connections between apps. -Let's start from our previous example. Consider the case where one wants to apply optical calibration and binary morphological dilatation -following the undersampling. +One nice feature of pyotb is in-memory connection between apps. It relies on +the so-called [streaming](https://www.orfeo-toolbox.org/CookBook/C++/StreamingAndThreading.html) +mechanism of OTB, that enables to process huge images with a limited memory +footprint. + +pyotb allows to pass any application's output to another. This enables to +build pipelines composed of several applications. + +Let's start from our previous example. Consider the case where one wants to +resample the image, then apply optical calibration and binary morphological +dilatation. We can write the following code to build a pipeline that will +generate the output in an end-to-end fashion, without being limited with the +input image size or writing temporary files. -Using pyotb, you can pass the output of an app as input of another app : ```python import pyotb -resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'interpolator': 'linear', - 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5}) +resampled = pyotb.RigidTransformResample({ + 'in': 'my_image.tif', + 'interpolator': 'linear', + 'transform.type.id.scaley': 0.5, + 'transform.type.id.scalex': 0.5 +}) + +calibrated = pyotb.OpticalCalibration({ + 'in': resampled, + 'level': 'toa' +}) + +dilated = pyotb.BinaryMorphologicalOperation({ + 'in': calibrated, + 'out': 'output.tif', + 'filter': 'dilate', + 'structype': 'ball', + 'xradius': 3, + 'yradius': 3 +}) +``` -calibrated = pyotb.OpticalCalibration({'in': resampled, 'level': 'toa'}) +We just have built our first pipeline! At this point, it's all symbolic since +no computation has been performed. To trigger our pipeline, one must call the +`write()` method from the pipeline termination: -dilated = pyotb.BinaryMorphologicalOperation({'in': calibrated, 'out': 'output.tif', 'filter': 'dilate', - 'structype': 'ball', 'xradius': 3, 'yradius': 3}) -dilated.write('result.tif') +```python +dilated.write('output.tif') ``` +In the next section, we will detail how `write()` works. + ## Writing the result of an app -Any pyotb object can be written to disk using the `write` method, e.g. : + +Any pyotb object can be written to disk using `write()`. + +Let's consider the following pyotb application instance: ```python import pyotb +resampled = pyotb.RigidTransformResample({ + 'in': 'my_image.tif', + 'interpolator': 'linear', + 'transform.type.id.scaley': 0.5, + 'transform.type.id.scalex': 0.5 +}) +``` + +We can then write the output of `resampled` as following: -resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'interpolator': 'linear', - 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5}) -# Here you can set optionally pixel type and extended filename variables -resampled.write({'out': 'output.tif'}, pixel_type='uint16', filename_extension='?nodata=65535') +```python +resampled.write('output.tif') ``` -Another possibility for writing results is to set the output parameter when initializing the application: +!!! note + + For applications that have multiple outputs, passing a `dict` of filenames + can be considered. Let's take the example of `MeanShiftSmoothing` which + has 2 output images: + + ```python + import pyotb + meanshift = pyotb.MeanShiftSmoothing('my_image.tif') + meanshift.write({'fout': 'output_1.tif', 'foutpos': 'output_2.tif'}) + ``` + +Another possibility for writing results is to set the output parameter when +initializing the application: + ```python import pyotb -resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'interpolator': 'linear', 'out': 'output.tif', - 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5}) -# Here you can set optionally pixel type and extended filename variables -resampled.write(pixel_type='uint16', filename_extension='?nodata=65535') -``` \ No newline at end of file +resampled = pyotb.RigidTransformResample({ + 'in': 'my_image.tif', + 'interpolator': 'linear', + 'out': 'output.tif', + 'transform.type.id.scaley': 0.5, + 'transform.type.id.scalex': 0.5 +}) +``` + +### Pixel type + +Setting the pixel type is optional, and can be achieved setting the +`pixel_type` argument: + +```python +resampled.write('output.tif', pixel_type='uint16') +``` + +The value of `pixel_type` corresponds to the name of a pixel type from OTB +applications (e.g. `'uint8'`, `'float'`, etc). + +### Extended filenames + +Extended filenames can be passed as `str` or `dict`. + +As `str`: + +```python +resampled.write( + ... + ext_fname='nodata=65535&box=0:0:256:256' +) +``` + +As `dict`: + +```python +resampled.write( + ... + ext_fname={'nodata': '65535', 'box': '0:0:256:256'} +) +``` + +!!! info + + When `ext_fname` is provided and the output filenames contain already some + extended filename pattern, the ones provided in the filenames take + priority over the ones passed in `ext_fname`. This allows to fine-grained + tune extended filenames for each output, with a common extended filenames + keys/values basis. diff --git a/doc/stylesheets/extra.css b/doc/stylesheets/extra.css deleted file mode 100644 index c67f70f85c74bbb5ffdb34443d58881964a6611a..0000000000000000000000000000000000000000 --- a/doc/stylesheets/extra.css +++ /dev/null @@ -1,4 +0,0 @@ -/* this is for readthedocs theme */ -.wy-nav-content { - max-width: 1000px; -} \ No newline at end of file diff --git a/doc/summarize.md b/doc/summarize.md new file mode 100644 index 0000000000000000000000000000000000000000..13f30214b9e5c3a53962e6e3765ea40f2388423a --- /dev/null +++ b/doc/summarize.md @@ -0,0 +1,167 @@ +## Summarize applications + +pyotb enables to summarize applications as a dictionary with keys/values for +parameters. This feature can be used to keep track of a process, composed of +multiple applications chained together. + +### Single application + +Let's take the example of one single application. + +```python +import pyotb + +app = pyotb.RigidTransformResample({ + 'in': 'my_image.tif', + 'interpolator': 'linear', + 'transform.type.id.scaley': 0.5, + 'transform.type.id.scalex': 0.5 +}) +``` + +The application can be summarized using `pyotb.summarize()` or +`app.summary()`, which are equivalent. + +```python +print(app.summarize()) +``` + +Results in the following (lines have been pretty printed for the sake of +documentation): + +```json lines +{ + 'name': 'RigidTransformResample', + 'parameters': { + 'transform.type': 'id', + 'in': 'my_image.tif', + 'interpolator': 'linear', + 'transform.type.id.scaley': 0.5, + 'transform.type.id.scalex': 0.5 + } +} +``` + +Note that we can also summarize an application after it has been executed: + +```python +app.write('output.tif', pixel_type='uint16') +print(app.summarize()) +``` + +Which results in the following: + +```json lines +{ + 'name': 'RigidTransformResample', + 'parameters': { + 'transform.type': 'id', + 'in': 'my_image.tif', + 'interpolator': 'linear', + 'transform.type.id.scaley': 0.5, + 'transform.type.id.scalex': 0.5, + 'out': 'output.tif' + } +} +``` + +Now `'output.tif'` has been added to the application parameters. + +### Multiple applications chained together (pipeline) + +When multiple applications are chained together, the summary of the last +application will describe all upstream processes. + +```python +import pyotb + +app1 = pyotb.RigidTransformResample({ + 'in': 'my_image.tif', + 'interpolator': 'linear', + 'transform.type.id.scaley': 0.5, + 'transform.type.id.scalex': 0.5 +}) +app2 = pyotb.Smoothing(app1) +print(app2.summarize()) +``` + +Results in: + +```json lines +{ + 'name': 'Smoothing', + 'parameters': { + 'type': 'anidif', + 'type.anidif.timestep': 0.125, + 'type.anidif.nbiter': 10, + 'type.anidif.conductance': 1.0, + 'in': { + 'name': 'RigidTransformResample', + 'parameters': { + 'transform.type': 'id', + 'in': 'my_image.tif', + 'interpolator': 'linear', + 'transform.type.id.scaley': 0.5, + 'transform.type.id.scalex': 0.5 + } + } + } +} +``` + +### Remote files URL stripping + +Cloud-based raster URLs often include tokens or random strings resulting from +the URL signing. +Those can be removed from the summarized paths, using the `strip_inpath` +and/or `strip_outpath` arguments respectively for inputs and/or outputs. + +Here is an example with Microsoft Planetary Computer: + +```python +import planetary_computer +import pyotb + +url = ( + "https://sentinel2l2a01.blob.core.windows.net/sentinel2-l2/31/N/EA/2023/" + "11/03/S2A_MSIL2A_20231103T095151_N0509_R079_T31NEA_20231103T161409.SAFE/" + "GRANULE/L2A_T31NEA_A043691_20231103T100626/IMG_DATA/R10m/T31NEA_20231103" + "T095151_B02_10m.tif" +) +signed_url = planetary_computer.sign_inplace(url) +app = pyotb.Smoothing(signed_url) +``` + +By default, the summary does not strip the URL. + +```python +print(app.summarize()["parameters"]["in"]) +``` + +This results in: + +``` +/vsicurl/https://sentinel2l2a01.blob.core.windows.net/sentinel2-l2/31/N/EA/... +2023/11/03/S2A_MSIL2A_20231103T095151_N0509_R079_T31NEA_20231103T161409.SAFE... +/GRANULE/L2A_T31NEA_A043691_20231103T100626/IMG_DATA/R10m/T31NEA_20231103T... +095151_B02_10m.tif?st=2023-11-07T15%3A52%3A47Z&se=2023-11-08T16%3A37%3A47Z&... +sp=rl&sv=2021-06-08&sr=c&skoid=c85c15d6-d1ae-42d4-af60-e2ca0f81359b&sktid=... +72f988bf-86f1-41af-91ab-2d7cd011db47&skt=2023-11-08T11%3A41%3A41Z&ske=2023-... +11-15T11%3A41%3A41Z&sks=b&skv=2021-06-08&sig=xxxxxxxxxxx...xxxxx +``` + +Now we can strip the URL to keep only the resource identifier and get rid of +the token: + +```python +print(app.summarize(strip_inpath=True)["parameters"]["in"]) +``` + +Which now results in: + +``` +/vsicurl/https://sentinel2l2a01.blob.core.windows.net/sentinel2-l2/31/N/EA/... +2023/11/03/S2A_MSIL2A_20231103T095151_N0509_R079_T31NEA_20231103T161409.SAFE... +/GRANULE/L2A_T31NEA_A043691_20231103T100626/IMG_DATA/R10m/T31NEA_20231103T... +095151_B02_10m.tif +``` \ No newline at end of file diff --git a/doc/troubleshooting.md b/doc/troubleshooting.md index e6b0b4c375e6c862d31b37d636cb86047ce36700..9cf5b84eff67fba64b7edb622a4214d87a562578 100644 --- a/doc/troubleshooting.md +++ b/doc/troubleshooting.md @@ -1,14 +1,42 @@ -## Troubleshooting: Known limitations +# Troubleshooting -### Failure of intermediate writing +## Migration from pyotb 1.5.4 (oct 2022) to 2.x.y -When chaining applications in-memory, there may be some problems when writing intermediate results, depending on the order +List of breaking changes: + +- `otbObject` has ben renamed `OTBObject` +- `otbObject.get_infos()` has been renamed `OTBObject.get_info()` +- `otbObject.key_output_image` has been renamed `OTBObject.output_image_key` +- `otbObject.key_input_image` has been renamed `OTBObject.input_image_key` +- `otbObject.read_values_at_coords()` has been renamed `OTBObject.get_values_at_coords()` +- `otbObject.xy_to_rowcol()` has been renamed `OTBObject.get_rowcol_from_xy()` +- `App.output_param` has been replaced with `App.output_image_key` +- `App.write()` argument `filename_extension` has been renamed `ext_fname` +- `App.save_objects()` has been renamed `App.__sync_parameters()` +- use `pyotb_app['paramname']` or `pyotb_app.app.GetParameterValue('paramname')` instead of `pyotb_app.GetParameterValue('paramname')` to access parameter `paramname` value +- use `pyotb_app['paramname']` instead of `pyotb_app.paramname` to access parameter `paramname` value +- `Output.__init__()` arguments `app` and `output_parameter_key` have been renamed `pyotb_app` and `param_key` +- `Output.pyotb_app` has been renamed `Output.parent_pyotb_app` +- `logicalOperation` has been renamed `LogicalOperation` + +## Known limitations with old versions + +!!! note + + All defects described below have been fixed since OTB 8.1.2 and pyotb 2.0.0 + +### Failure of intermediate writing (otb < 8.1, pyotb < 1.5.4) + +When chaining applications in-memory, there may be some problems when writing +intermediate results, depending on the order the writings are requested. Some examples can be found below: #### Example of failures involving slicing -For some applications (non-exhaustive know list: OpticalCalibration, DynamicConvert, BandMath), we can face unexpected -failures when using channels slicing +For some applications (non-exhaustive know list: OpticalCalibration, +DynamicConvert, BandMath), we can face unexpected failures when using channels +slicing + ```python import pyotb @@ -28,6 +56,7 @@ one_band.write('one_band.tif') # Failure here ``` When writing is triggered right after the application declaration, no problem occurs: + ```python import pyotb @@ -51,11 +80,10 @@ inp.write('stretched.tif') one_band.write('one_band.tif') ``` - #### Example of failures involving arithmetic operation -One can meet errors when using arithmetic operations at the end of a pipeline when DynamicConvert, BandMath or -OpticalCalibration is involved: +One can meet errors when using arithmetic operations at the end of a pipeline +when DynamicConvert, BandMath or OpticalCalibration is involved: ```python import pyotb diff --git a/mkdocs.yml b/mkdocs.yml index 56542796967f59adaed72babd6de011ec9afd6cd..88589b11d42843d02b111f71b1d0e20546412d3e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -55,21 +55,27 @@ extra: - icon: fontawesome/brands/gitlab link: https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb extra_css: - - stylesheets/extra.css + - https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/8.1.2-rc1/Documentation/Cookbook/_static/css/otb_theme.css + - extra.css use_directory_urls: false # this creates some pyotb/core.html pages instead of pyotb/core/index.html markdown_extensions: + - admonition - toc: permalink: true title: On this page toc_depth: 1-2 - pymdownx.highlight: anchor_linenums: true - - pymdownx.inlinehilite - - pymdownx.snippets + - pymdownx.details + - pymdownx.superfences: + custom_fences: + - name: python + class: python + format: !!python/name:pymdownx.superfences.fence_code_format -# Rest of the navigation.. -site_name: "pyotb documentation: a Python extension of OTB" +# Rest of the navigation. +site_name: "pyotb: Orfeo ToolBox for Python" repo_url: https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb repo_name: pyotb docs_dir: doc/ diff --git a/pyotb/__init__.py b/pyotb/__init__.py index 187b64a3eab5232464472d46eeca1e1f0ca9f161..5b455abe8a07775d7b5e865053e6641867bd246d 100644 --- a/pyotb/__init__.py +++ b/pyotb/__init__.py @@ -1,8 +1,25 @@ # -*- coding: utf-8 -*- """This module provides convenient python wrapping of otbApplications.""" -__version__ = "1.5.4" +__version__ = "2.0.0" -from .apps import * -from .core import App, Output, Input, get_nbchannels, get_pixel_type -from .functions import * # pylint: disable=redefined-builtin +from .install import install_otb from .helpers import logger, set_logger_level +from .core import ( + OTBObject, + App, + Input, + Output, + get_nbchannels, + get_pixel_type, + summarize, +) +from .apps import * + +from .functions import ( # pylint: disable=redefined-builtin + all, + any, + clip, + define_processing_area, + run_tf_function, + where, +) diff --git a/pyotb/apps.py b/pyotb/apps.py index e5287bbd21a30326b0c9a8341d1d9bb0fa74d3f4..45e49c4788837bfce7bc09091cd2c976a02aa8b3 100644 --- a/pyotb/apps.py +++ b/pyotb/apps.py @@ -1,85 +1,38 @@ -# -*- coding: utf-8 -*- """Search for OTB (set env if necessary), subclass core.App for each available application.""" +from __future__ import annotations + import os -import sys -from pathlib import Path -from .helpers import logger, find_otb +import otbApplication as otb # pylint: disable=import-error -otb = find_otb() +from .core import App +from .helpers import logger -def get_available_applications(as_subprocess=False): +def get_available_applications() -> tuple[str]: """Find available OTB applications. - Args: - as_subprocess: indicate if function should list available applications using subprocess call - Returns: tuple of available applications + Raises: + SystemExit: if no application is found + """ - app_list = () - if as_subprocess and sys.executable: - # Currently, there is an incompatibility between OTBTF and Tensorflow that causes segfault - # when OTBTF apps are used in a script where tensorflow has already been imported. - # See https://github.com/remicres/otbtf/issues/28 - # Thus, we run this piece of code in a clean independent `subprocess` that doesn't interact with Tensorflow - env = os.environ.copy() - if "PYTHONPATH" not in env: - env["PYTHONPATH"] = "" - env["PYTHONPATH"] = ":" + str(Path(otb.__file__).parent) - env["OTB_LOGGER_LEVEL"] = "CRITICAL" # in order to suppress warnings while listing applications - pycmd = "import otbApplication; print(otbApplication.Registry.GetAvailableApplications())" - cmd_args = [sys.executable, "-c", pycmd] - - try: - import subprocess # pylint: disable=import-outside-toplevel - params = {"env": env, "stdout": subprocess.PIPE, "stderr": subprocess.PIPE} - with subprocess.Popen(cmd_args, **params) as p: - logger.debug('Exec "%s \'%s\'"', ' '.join(cmd_args[:-1]), pycmd) - stdout, stderr = p.communicate() - stdout, stderr = stdout.decode(), stderr.decode() - # ast.literal_eval is secure and will raise more handy Exceptions than eval - from ast import literal_eval # pylint: disable=import-outside-toplevel - app_list = literal_eval(stdout.strip()) - assert isinstance(app_list, (tuple, list)) - except subprocess.SubprocessError: - logger.debug("Failed to call subprocess") - except (ValueError, SyntaxError, AssertionError): - logger.debug("Failed to decode output or convert to tuple:\nstdout=%s\nstderr=%s", stdout, stderr) - - if not app_list: - logger.info("Failed to list applications in an independent process. Falling back to local python import") - # Find applications using the normal way - if not app_list: - app_list = otb.Registry.GetAvailableApplications() - if not app_list: - logger.warning("Unable to load applications. Set env variable OTB_APPLICATION_PATH then try again") - return () - - logger.info("Successfully loaded %s OTB applications", len(app_list)) - return app_list - - -AVAILABLE_APPLICATIONS = get_available_applications(as_subprocess=True) - -# First core.py call (within __init__ scope) -from .core import App # pylint: disable=wrong-import-position - -# This is to enable aliases of Apps, i.e. using apps like `pyotb.AppName(...)` instead of `pyotb.App("AppName", ...)` -_CODE_TEMPLATE = """ -class {name}(App): - """ """ - def __init__(self, *args, **kwargs): - super().__init__('{name}', *args, **kwargs) -""" + app_list = otb.Registry.GetAvailableApplications() + if app_list: + logger.info("Successfully loaded %s OTB applications", len(app_list)) + return app_list + raise SystemExit( + "Unable to load applications. Set env variable OTB_APPLICATION_PATH and try again." + ) class OTBTFApp(App): - """Helper for OTBTF.""" + """Helper for OTBTF to ensure the nb_sources variable is set.""" + @staticmethod - def set_nb_sources(*args, n_sources=None): + def set_nb_sources(*args, n_sources: int = None): """Set the number of sources of TensorflowModelServe. Can be either user-defined or deduced from the args. Args: @@ -89,52 +42,46 @@ class OTBTFApp(App): """ if n_sources: - os.environ['OTB_TF_NSOURCES'] = str(int(n_sources)) + os.environ["OTB_TF_NSOURCES"] = str(int(n_sources)) else: # Retrieving the number of `source#.il` parameters - params_dic = {k: v for arg in args if isinstance(arg, dict) for k, v in arg.items()} - n_sources = len([k for k in params_dic if 'source' in k and k.endswith('.il')]) + params_dic = { + k: v for arg in args if isinstance(arg, dict) for k, v in arg.items() + } + n_sources = len( + [k for k in params_dic if "source" in k and k.endswith(".il")] + ) if n_sources >= 1: - os.environ['OTB_TF_NSOURCES'] = str(n_sources) + os.environ["OTB_TF_NSOURCES"] = str(n_sources) - def __init__(self, app_name, *args, n_sources=None, **kwargs): + def __init__(self, name: str, *args, n_sources: int = None, **kwargs): """Constructor for an OTBTFApp object. Args: - app_name: name of the OTBTF app - *args: arguments (dict). NB: we don't need kwargs because it cannot contain source#.il + name: name of the OTBTF app n_sources: number of sources. Default is None (resolves the number of sources based on the content of the dict passed in args, where some 'source' str is found) - **kwargs: kwargs + """ self.set_nb_sources(*args, n_sources=n_sources) - super().__init__(app_name, *args, **kwargs) + super().__init__(name, *args, **kwargs) -for _app in AVAILABLE_APPLICATIONS: - # Customize the behavior for some OTBTF applications. The user doesn't need to set the env variable - # `OTB_TF_NSOURCES`, it is handled in pyotb - if _app == 'TensorflowModelServe': - class TensorflowModelServe(OTBTFApp): - """Serve a Tensorflow model using OTBTF.""" - def __init__(self, *args, n_sources=None, **kwargs): - """Constructor for a TensorflowModelServe object.""" - super().__init__('TensorflowModelServe', *args, n_sources=n_sources, **kwargs) - - elif _app == 'PatchesExtraction': - class PatchesExtraction(OTBTFApp): - """Extract patches using OTBTF.""" - def __init__(self, *args, n_sources=None, **kwargs): - """Constructor for a PatchesExtraction object.""" - super().__init__('PatchesExtraction', *args, n_sources=n_sources, **kwargs) - - elif _app == 'TensorflowModelTrain': - class TensorflowModelTrain(OTBTFApp): - """Train a Tensorflow model using OTBTF.""" - def __init__(self, *args, n_sources=None, **kwargs): - """Constructor for a TensorflowModelTrain object.""" - super().__init__('TensorflowModelTrain', *args, n_sources=n_sources, **kwargs) +AVAILABLE_APPLICATIONS = get_available_applications() +# This is to enable aliases of Apps, i.e. `pyotb.AppName(...)` instead of `pyotb.App("AppName", ...)` +_CODE_TEMPLATE = """ +class {name}(App): + def __init__(self, *args, **kwargs): + super().__init__('{name}', *args, **kwargs) +""" + +for _app in AVAILABLE_APPLICATIONS: + # Customize the behavior for some OTBTF applications. `OTB_TF_NSOURCES` is now handled by pyotb + if _app in ("PatchesExtraction", "TensorflowModelTrain", "TensorflowModelServe"): + exec( # pylint: disable=exec-used + _CODE_TEMPLATE.format(name=_app).replace("(App)", "(OTBTFApp)") + ) # Default behavior for any OTB application else: exec(_CODE_TEMPLATE.format(name=_app)) # pylint: disable=exec-used diff --git a/pyotb/core.py b/pyotb/core.py index 5b9a895b35e342bc16af2b5ab7334b269fa33e49..9cd9d4eeff26ce85ccd1eb63ffef30b3445e3935 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1,480 +1,371 @@ -# -*- coding: utf-8 -*- """This module is the core of pyotb.""" +from __future__ import annotations + +import re +from abc import ABC, abstractmethod +from ast import literal_eval from pathlib import Path +from time import perf_counter +from typing import Any import numpy as np -import otbApplication as otb +import otbApplication as otb # pylint: disable=import-error from .helpers import logger +from .depreciation import deprecated_alias, depreciation_warning, deprecated_attr -class otbObject: - """Base class that gathers common operations for any OTB in-memory raster.""" - _name = "" - app = None - output_param = "" +class OTBObject(ABC): + """Abstraction of an image object, for a whole app or one specific output.""" @property - def name(self): - """Application name that will be printed in logs. + @abstractmethod + def name(self) -> str: + """Application name by default, but a custom name may be passed during init.""" - Returns: - user's defined name or appname + @property + @abstractmethod + def app(self) -> otb.Application: + """Reference to the otb.Application instance linked to this object.""" - """ - return self._name or self.app.GetName() + @property + @abstractmethod + def output_image_key(self) -> str: + """Return the name of a parameter key associated to the main output image of the object.""" + + @property + @deprecated_attr(replacement="output_image_key") + def output_param(self) -> str: + """Return the name of a parameter key associated to the main output image of the object (deprecated).""" - @name.setter - def name(self, val): - """Set custom name. + @property + @abstractmethod + def exports_dic(self) -> dict[str, dict]: + """Ref to an internal dict of np.array exports, to avoid duplicated ExportImage().""" - Args: - val: new name + @property + def metadata(self) -> dict[str, (str, float, list[float])]: + """Return image metadata as dictionary. + + The returned dict results from the concatenation of the first output + image metadata dictionary and the metadata dictionary. """ - self._name = val + # Image Metadata + otb_imd = self.app.GetImageMetadata(self.output_image_key) + cats = ["Num", "Str", "L1D", "Time"] + imd = { + key: getattr(otb_imd, f"get_{cat.lower()}")(key) + for cat in cats + for key in getattr(otb_imd, f"GetKeyList{cat}")().split(" ") + if getattr(otb_imd, "has")(key) + } + + # Other metadata dictionary: key-value pairs parsing is required + mdd = dict(self.app.GetMetadataDictionary(self.output_image_key)) + new_mdd = {} + for key, val in mdd.items(): + new_key = key + new_val = val + if isinstance(val, str): + splits = val.split("=") + if key.lower().startswith("metadata_") and len(splits) == 2: + new_key = splits[0].strip() + new_val = splits[1].strip() + new_mdd[new_key] = new_val + + return {**new_mdd, **imd} @property - def dtype(self): - """Expose the pixel type of an output image using numpy convention. + def dtype(self) -> np.dtype: + """Expose the pixel type of output image using numpy convention. Returns: dtype: pixel type of the output image """ - try: - enum = self.app.GetParameterOutputImagePixelType(self.output_param) - return self.app.ConvertPixelTypeToNumpy(enum) - except RuntimeError: - return None + enum = self.app.GetParameterOutputImagePixelType(self.output_image_key) + return self.app.ConvertPixelTypeToNumpy(enum) @property - def shape(self): + def shape(self) -> tuple[int]: """Enables to retrieve the shape of a pyotb object using numpy convention. Returns: shape: (height, width, bands) """ - width, height = self.app.GetImageSize(self.output_param) - bands = self.app.GetImageNbBands(self.output_param) - return (height, width, bands) - - def write(self, *args, filename_extension="", pixel_type=None, **kwargs): - """Trigger execution, set output pixel type and write the output. - - Args: - *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key contains - non-standard characters such as a point, e.g. {'io.out':'output.tif'} - - string, useful when there is only one output, e.g. 'output.tif' - - None if output file was passed during App init - filename_extension: Optional, an extended filename as understood by OTB (e.g. "&gdal:co:TILED=YES") - Will be used for all outputs (Default value = "") - pixel_type: Can be : - dictionary {output_parameter_key: pixeltype} when specifying for several outputs - - str (e.g. 'uint16') or otbApplication.ImagePixelType_... When there are several - outputs, all outputs are written with this unique type. - Valid pixel types are uint8, uint16, uint32, int16, int32, float, double, - cint16, cint32, cfloat, cdouble. (Default value = None) - **kwargs: keyword arguments e.g. out='output.tif' - """ - # Gather all input arguments in kwargs dict - for arg in args: - if isinstance(arg, dict): - kwargs.update(arg) - elif isinstance(arg, str) and kwargs: - logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, arg) - elif isinstance(arg, str): - kwargs.update({self.output_param: arg}) - - dtypes = {} - if isinstance(pixel_type, dict): - dtypes = {k: parse_pixel_type(v) for k, v in pixel_type.items()} - elif pixel_type is not None: - typ = parse_pixel_type(pixel_type) - if isinstance(self, App): - dtypes = {key: typ for key in self.output_parameters_keys} - elif isinstance(self, otbObject): - dtypes = {self.output_param: typ} - - if filename_extension: - logger.debug('%s: using extended filename for outputs: %s', self.name, filename_extension) - if not filename_extension.startswith('?'): - filename_extension = "?" + filename_extension - - # Case output parameter was set during App init - if not kwargs: - if self.output_param in self.parameters: - if dtypes: - self.app.SetParameterOutputImagePixelType(self.output_param, dtypes[self.output_param]) - if filename_extension: - new_val = self.parameters[self.output_param] + filename_extension - self.app.SetParameterString(self.output_param, new_val) - else: - raise ValueError(f'{self.app.GetName()}: Output parameter is missing.') - - # Parse kwargs - for key, output_filename in kwargs.items(): - # Stop process if a bad parameter is given - if key not in self.app.GetParametersKeys(): - raise KeyError(f'{self.app.GetName()}: Unknown parameter key "{key}"') - # Check if extended filename was not provided twice - if '?' in output_filename and filename_extension: - logger.warning('%s: extended filename was provided twice. Using the one found in path.', self.name) - elif filename_extension: - output_filename += filename_extension - - logger.debug('%s: "%s" parameter is %s', self.name, key, output_filename) - self.app.SetParameterString(key, output_filename) - - if key in dtypes: - self.app.SetParameterOutputImagePixelType(key, dtypes[key]) - - logger.debug('%s: flushing data to disk', self.name) - try: - self.app.WriteOutput() - except RuntimeError: - logger.debug('%s: failed to simply write output, executing once again then writing', self.name) - self.app.ExecuteAndWriteOutput() + width, height = self.app.GetImageSize(self.output_image_key) + bands = self.app.GetImageNbBands(self.output_image_key) + return height, width, bands - def to_numpy(self, preserve_dtype=True, copy=False): - """Export a pyotb object to numpy array. + @property + def transform(self) -> tuple[int]: + """Get image affine transform, rasterio style. - Args: - preserve_dtype: when set to True, the numpy array is created with the same pixel type as - the otbObject first output. Default is True. - copy: whether to copy the output array, default is False - required to True if preserve_dtype is False and the source app reference is lost + See https://www.perrygeo.com/python-affine-transforms.html Returns: - a numpy array - + transform: (X spacing, X offset, X origin, Y offset, Y spacing, Y origin) """ - array = self.app.ExportImage(self.output_param)['array'] - if preserve_dtype: - return array.astype(self.dtype) - if copy: - return array.copy() - return array - - def to_rasterio(self): - """Export image as a numpy array and its metadata compatible with rasterio. - - Returns: - array : a numpy array in the (bands, height, width) order - profile: a metadata dict required to write image using rasterio - - """ - array = self.to_numpy(preserve_dtype=True, copy=False) - array = np.moveaxis(array, 2, 0) - proj = self.app.GetImageProjection(self.output_param) - spacing_x, spacing_y = self.app.GetImageSpacing(self.output_param) - origin_x, origin_y = self.app.GetImageOrigin(self.output_param) + spacing_x, spacing_y = self.app.GetImageSpacing(self.output_image_key) + origin_x, origin_y = self.app.GetImageOrigin(self.output_image_key) # Shift image origin since OTB is giving coordinates of pixel center instead of corners origin_x, origin_y = origin_x - spacing_x / 2, origin_y - spacing_y / 2 - profile = { - 'crs': proj, 'dtype': array.dtype, - 'count': array.shape[0], 'height': array.shape[1], 'width': array.shape[2], - 'transform': (spacing_x, 0.0, origin_x, 0.0, spacing_y, origin_y) # here we force pixel rotation to 0 ! - } - return array, profile + return spacing_x, 0.0, origin_x, 0.0, spacing_y, origin_y - # Special methods - def __getitem__(self, key): - """Override the default __getitem__ behaviour. - - This function enables 2 things : - - access attributes like that : object['any_attribute'] - - slicing, i.e. selecting ROI/bands. For example, selecting first 3 bands: object[:, :, :3] - selecting bands 1, 2 & 5 : object[:, :, [0, 1, 4]] - selecting 1000x1000 subset : object[:1000, :1000] + def summarize(self, *args, **kwargs): + """Recursively summarize an app parameters and its parents. Args: - key: attribute key + *args: args for `pyotb.summarize()` + **kwargs: keyword args for `pyotb.summarize()` Returns: - attribute or Slicer - """ - # Accessing string attributes - if isinstance(key, str): - return self.__dict__.get(key) - # Slicing - if not isinstance(key, tuple) or (isinstance(key, tuple) and (len(key) < 2 or len(key) > 3)): - raise ValueError(f'"{key}"cannot be interpreted as valid slicing. Slicing should be 2D or 3D.') - if isinstance(key, tuple) and len(key) == 2: - # Adding a 3rd dimension - key = key + (slice(None, None, None),) - (rows, cols, channels) = key - return Slicer(self, rows, cols, channels) - - def __getattr__(self, name): - """This method is called when the default attribute access fails. - - We choose to access the attribute `name` of self.app. - Thus, any method of otbApplication can be used transparently on otbObject objects, - e.g. SetParameterOutputImagePixelType() or ExportImage() work - - Args: - name: attribute name - - Returns: - attribute - - Raises: - AttributeError: when `name` is not an attribute of self.app - - """ - try: - res = getattr(self.app, name) - return res - except AttributeError as e: - raise AttributeError(f'{self.name}: could not find attribute `{name}`') from e - - def __add__(self, other): - """Overrides the default addition and flavours it with BandMathX. - - Args: - other: the other member of the operation - - Returns: - self + other - - """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return Operation('+', self, other) - - def __sub__(self, other): - """Overrides the default subtraction and flavours it with BandMathX. - - Args: - other: the other member of the operation - - Returns: - self - other + app summary, same as `pyotb.summarize()` """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return Operation('-', self, other) - - def __mul__(self, other): - """Overrides the default subtraction and flavours it with BandMathX. + return summarize(self, *args, **kwargs) - Args: - other: the other member of the operation - - Returns: - self * other + def get_info(self) -> dict[str, (str, float, list[float])]: + """Return a dict output of ReadImageInfo for the first image output.""" + return App("ReadImageInfo", self, quiet=True).data - """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return Operation('*', self, other) + def get_statistics(self) -> dict[str, (str, float, list[float])]: + """Return a dict output of ComputeImagesStatistics for the first image output.""" + return App("ComputeImagesStatistics", self, quiet=True).data - def __truediv__(self, other): - """Overrides the default subtraction and flavours it with BandMathX. + def get_values_at_coords( + self, row: int, col: int, bands: int | list[int] = None + ) -> list[float] | float: + """Get pixel value(s) at a given YX coordinates. Args: - other: the other member of the operation + row: index along Y / latitude axis + col: index along X / longitude axis + bands: band number(s) to fetch values from Returns: - self / other - - """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return Operation('/', self, other) - - def __radd__(self, other): - """Overrides the default reverse addition and flavours it with BandMathX. + single numerical value or a list of values for each band - Args: - other: the other member of the operation - - Returns: - other + self + Raises: + TypeError: if bands is not a slice or list """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return Operation('+', other, self) - - def __rsub__(self, other): - """Overrides the default subtraction and flavours it with BandMathX. + channels = [] + app = App("PixelValue", self, coordx=col, coordy=row, frozen=True, quiet=True) + if bands is not None: + if isinstance(bands, int): + if bands < 0: + bands = self.shape[2] + bands + channels = [bands] + elif isinstance(bands, slice): + channels = self.channels_list_from_slice(bands) + elif not isinstance(bands, list): + raise TypeError( + f"{self.name}: type '{type(bands)}' cannot be interpreted as a valid slicing" + ) + if channels: + app.app.Execute() + app.set_parameters({"cl": [f"Channel{n + 1}" for n in channels]}) + app.execute() + data = literal_eval(app.app.GetParameterString("value")) + return data[0] if len(channels) == 1 else data + + def channels_list_from_slice(self, bands: slice) -> list[int]: + """Get list of channels to read values at, from a slice. Args: - other: the other member of the operation + bands: slice obtained when using app[:] Returns: - other - self + list of channels to select - """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return Operation('-', other, self) - - def __rmul__(self, other): - """Overrides the default multiplication and flavours it with BandMathX. - - Args: - other: the other member of the operation - - Returns: - other * self + Raises: + ValueError: if the slice is malformed """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return Operation('*', other, self) - - def __rtruediv__(self, other): - """Overrides the default division and flavours it with BandMathX. + nb_channels = self.shape[2] + start, stop, step = bands.start, bands.stop, bands.step + start = nb_channels + start if isinstance(start, int) and start < 0 else start + stop = nb_channels + stop if isinstance(stop, int) and stop < 0 else stop + step = 1 if step is None else step + if start is not None and stop is not None: + return list(range(start, stop, step)) + if start is not None and stop is None: + return list(range(start, nb_channels, step)) + if start is None and stop is not None: + return list(range(0, stop, step)) + if start is None and stop is None: + return list(range(0, nb_channels, step)) + raise ValueError( + f"{self.name}: '{bands}' cannot be interpreted as valid slicing." + ) + + def export( + self, key: str = None, preserve_dtype: bool = True + ) -> dict[str, dict[str, np.ndarray]]: + """Export a specific output image as numpy array and store it in object exports_dic. Args: - other: the other member of the operation - - Returns: - other / self - - """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return Operation('/', other, self) - - def __abs__(self): - """Overrides the default abs operator and flavours it with BandMathX. + key: parameter key to export, if None then the default one will be used + preserve_dtype: convert the array to the same pixel type as the App first output Returns: - abs(self) + the exported numpy array """ - return Operation('abs', self) + if key is None: + key = self.output_image_key + if key not in self.exports_dic: + self.exports_dic[key] = self.app.ExportImage(key) + if preserve_dtype: + self.exports_dic[key]["array"] = self.exports_dic[key]["array"].astype( + self.dtype + ) + return self.exports_dic[key] + + def to_numpy( + self, key: str = None, preserve_dtype: bool = True, copy: bool = False + ) -> np.ndarray: + """Export a pyotb object to numpy array. - def __ge__(self, other): - """Overrides the default greater or equal and flavours it with BandMathX. + A copy is avoided by default, but may be required if preserve_dtype is False + and the source app reference is lost. Args: - other: the other member of the operation + key: the output parameter name to export as numpy array + preserve_dtype: convert the array to the same pixel type as the App first output + copy: whether to copy the output array instead of returning a reference Returns: - self >= other + a numpy array that may already have been cached in self.exports_dic """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return logicalOperation('>=', self, other) - - def __le__(self, other): - """Overrides the default less or equal and flavours it with BandMathX. + data = self.export(key, preserve_dtype) + return data["array"].copy() if copy else data["array"] - Args: - other: the other member of the operation + def to_rasterio(self) -> tuple[np.ndarray, dict[str, Any]]: + """Export image as a numpy array and its metadata compatible with rasterio. Returns: - self <= other + array : a numpy array in the (bands, height, width) order + profile: a metadata dict required to write image using rasterio """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return logicalOperation('<=', self, other) + profile = {} + array = self.to_numpy(preserve_dtype=True, copy=False) + proj = self.app.GetImageProjection(self.output_image_key) + profile.update({"crs": proj, "dtype": array.dtype, "transform": self.transform}) + height, width, count = array.shape + profile.update({"count": count, "height": height, "width": width}) + return np.moveaxis(array, 2, 0), profile - def __gt__(self, other): - """Overrides the default greater operator and flavours it with BandMathX. + def get_rowcol_from_xy(self, x: float, y: float) -> tuple[int, int]: + """Find (row, col) index using (x, y) projected coordinates - image CRS is expected. Args: - other: the other member of the operation + x: longitude or projected X + y: latitude or projected Y Returns: - self > other + pixel index as (row, col) """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return logicalOperation('>', self, other) + spacing_x, _, origin_x, _, spacing_y, origin_y = self.transform + row, col = (origin_y - y) / spacing_y, (x - origin_x) / spacing_x + return abs(int(row)), int(col) - def __lt__(self, other): - """Overrides the default less operator and flavours it with BandMathX. + @staticmethod + def __create_operator(op_cls, name, x, y) -> Operation: + """Create an operator. Args: - other: the other member of the operation + op_cls: Operator class + name: operator expression + x: first element + y: second element Returns: - self < other + an Operation object instance """ - if isinstance(other, (np.ndarray, np.generic)): + if isinstance(y, (np.ndarray, np.generic)): return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return logicalOperation('<', self, other) + return op_cls(name, x, y) - def __eq__(self, other): - """Overrides the default eq operator and flavours it with BandMathX. + def __add__(self, other: OTBObject | str | float) -> Operation: + """Addition.""" + return self.__create_operator(Operation, "+", self, other) - Args: - other: the other member of the operation + def __sub__(self, other: OTBObject | str | float) -> Operation: + """Subtraction.""" + return self.__create_operator(Operation, "-", self, other) - Returns: - self == other + def __mul__(self, other: OTBObject | str | float) -> Operation: + """Multiplication.""" + return self.__create_operator(Operation, "*", self, other) - """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return logicalOperation('==', self, other) + def __truediv__(self, other: OTBObject | str | float) -> Operation: + """Division.""" + return self.__create_operator(Operation, "/", self, other) - def __ne__(self, other): - """Overrides the default different operator and flavours it with BandMathX. + def __radd__(self, other: OTBObject | str | float) -> Operation: + """Right addition.""" + return self.__create_operator(Operation, "+", other, self) - Args: - other: the other member of the operation + def __rsub__(self, other: OTBObject | str | float) -> Operation: + """Right subtraction.""" + return self.__create_operator(Operation, "-", other, self) - Returns: - self != other + def __rmul__(self, other: OTBObject | str | float) -> Operation: + """Right multiplication.""" + return self.__create_operator(Operation, "*", other, self) - """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return logicalOperation('!=', self, other) + def __rtruediv__(self, other: OTBObject | str | float) -> Operation: + """Right division.""" + return self.__create_operator(Operation, "/", other, self) - def __or__(self, other): - """Overrides the default or operator and flavours it with BandMathX. + def __abs__(self) -> Operation: + """Absolute value.""" + return Operation("abs", self) - Args: - other: the other member of the operation + def __ge__(self, other: OTBObject | str | float) -> LogicalOperation: + """Greater of equal than.""" + return self.__create_operator(LogicalOperation, ">=", self, other) - Returns: - self || other + def __le__(self, other: OTBObject | str | float) -> LogicalOperation: + """Lower of equal than.""" + return self.__create_operator(LogicalOperation, "<=", self, other) - """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return logicalOperation('||', self, other) + def __gt__(self, other: OTBObject | str | float) -> LogicalOperation: + """Greater than.""" + return self.__create_operator(LogicalOperation, ">", self, other) - def __and__(self, other): - """Overrides the default and operator and flavours it with BandMathX. + def __lt__(self, other: OTBObject | str | float) -> LogicalOperation: + """Lower than.""" + return self.__create_operator(LogicalOperation, "<", self, other) - Args: - other: the other member of the operation - - Returns: - self && other + def __eq__(self, other: OTBObject | str | float) -> LogicalOperation: + """Equality.""" + return self.__create_operator(LogicalOperation, "==", self, other) - """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return logicalOperation('&&', self, other) + def __ne__(self, other: OTBObject | str | float) -> LogicalOperation: + """Inequality.""" + return self.__create_operator(LogicalOperation, "!=", self, other) - # TODO: other operations ? - # e.g. __pow__... cf https://docs.python.org/3/reference/datamodel.html#emulating-numeric-types + def __or__(self, other: OTBObject | str | float) -> LogicalOperation: + """Logical or.""" + return self.__create_operator(LogicalOperation, "||", self, other) - def __hash__(self): - """Override the default behaviour of the hash function. + def __and__(self, other: OTBObject | str | float) -> LogicalOperation: + """Logical and.""" + return self.__create_operator(LogicalOperation, "&&", self, other) - Returns: - self hash - - """ - return id(self) + # Some other operations could be implemented with the same pattern + # e.g. __pow__... cf https://docs.python.org/3/reference/datamodel.html#emulating-numeric-types - def __array__(self): + def __array__(self) -> np.ndarray: """This is called when running np.asarray(pyotb_object). Returns: @@ -483,222 +374,580 @@ class otbObject: """ return self.to_numpy() - def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs) -> App: """This is called whenever a numpy function is called on a pyotb object. Operation is performed in numpy, then imported back to pyotb with the same georeference as input. + At least one obj is unputs has to be an OTBObject. Args: ufunc: numpy function method: an internal numpy argument - inputs: inputs, at least one being pyotb object. If there are several pyotb objects, they must all have - the same georeference and pixel size. + inputs: inputs, with equal shape in case of several images / OTBObject **kwargs: kwargs of the numpy function Returns: a pyotb object """ - if method == '__call__': + if method == "__call__": # Converting potential pyotb inputs to arrays arrays = [] image_dic = None for inp in inputs: if isinstance(inp, (float, int, np.ndarray, np.generic)): arrays.append(inp) - elif isinstance(inp, otbObject): - image_dic = inp.app.ExportImage(self.output_param) - array = image_dic['array'] - arrays.append(array) + elif isinstance(inp, OTBObject): + image_dic = inp.export() + arrays.append(image_dic["array"]) else: - print(type(self)) + logger.debug(type(self)) return NotImplemented - # Performing the numpy operation result_array = ufunc(*arrays, **kwargs) result_dic = image_dic - result_dic['array'] = result_array - - # Importing back to OTB - app = App('ExtractROI', frozen=True, image_dic=result_dic) # pass the result_dic just to keep reference + result_dic["array"] = result_array + # Importing back to OTB, pass the result_dic just to keep reference + pyotb_app = App("ExtractROI", frozen=True, quiet=True) if result_array.shape[2] == 1: - app.ImportImage('in', result_dic) + pyotb_app.app.ImportImage("in", result_dic) else: - app.ImportVectorImage('in', result_dic) - app.execute() - return app - + pyotb_app.app.ImportVectorImage("in", result_dic) + pyotb_app.execute() + return pyotb_app return NotImplemented - def summarize(self): - """Return a nested dictionary summarizing the otbObject. + def __hash__(self) -> int: + """Override the default behaviour of the hash function. Returns: - Nested dictionary summarizing the otbObject + self hash """ - params = self.parameters - for k, p in params.items(): - # In the following, we replace each parameter which is an otbObject, with its summary. - if isinstance(p, otbObject): # single parameter - params[k] = p.summarize() - elif isinstance(p, list): # parameter list - params[k] = [pi.summarize() if isinstance(pi, otbObject) else pi for pi in p] + return id(self) - return {"name": self.name, "parameters": params} + def __getattr__(self, item: str): + """Provides depreciation of old methods to access the OTB application values. + This function will be removed completely in future releases. -class App(otbObject): - """Class of an OTB app.""" - def __init__(self, appname, *args, frozen=False, quiet=False, - preserve_dtype=False, image_dic=None, **kwargs): - """Enables to init an OTB application as a oneliner. Handles in-memory connection between apps. + Args: + item: attribute name + + """ + note = ( + "Since pyotb 2.0.0, OTBObject instances have stopped to forward " + "attributes to their own internal otbApplication instance. " + "`App.app` can be used to call otbApplications methods." + ) + hint = None + + if item in dir(self.app): + hint = f"Maybe try `pyotb_app.app.{item}` instead of `pyotb_app.{item}`? " + if item.startswith("GetParameter"): + hint += ( + "Note: `pyotb_app.app.GetParameterValue('paramname')` can be " + "shorten with `pyotb_app['paramname']` to access parameters " + "values." + ) + elif item in self.parameters_keys: + # Because in pyotb 1.5.4, app outputs were added as instance attributes + hint = ( + "Note: `pyotb_app.paramname` is no longer supported. Starting " + "from pyotb 2.0.0, `pyotb_app['paramname']` can be used to " + "access parameters values. " + ) + if hint: + depreciation_warning(f"{note} {hint}") + raise AttributeError( + f"'{type(self).__name__}' object has no attribute '{item}'" + ) + + def __getitem__(self, key) -> Any | list[float] | float | Slicer: + """Override the default __getitem__ behaviour. + + This function enables 2 things : + - slicing, i.e. selecting ROI/bands + - access pixel value(s) at a specified row, col index Args: - appname: name of the app, e.g. 'Smoothing' - *args: used for passing application parameters. Can be : - - dictionary containing key-arguments enumeration. Useful when a key is python-reserved - (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit") - - string, App or Output, useful when the user wants to specify the input "in" - - list, useful when the user wants to specify the input list 'il' - frozen: freeze OTB app in order to use execute() later and avoid blocking process during __init___ - quiet: whether to print logs of the OTB app - preserve_dtype: propagate the pixel type from inputs to output. If several inputs, the type of an - arbitrary input is considered. If several outputs, all will have the same type. - image_dic: enables to keep a reference to image_dic. image_dic is a dictionary, such as - the result of app.ExportImage(). Use it when the app takes a numpy array as input. - See this related issue for why it is necessary to keep reference of object: - https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/1824 - **kwargs: used for passing application parameters. - e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif' + key: attribute key + + Returns: + list of pixel values if vector image, or pixel value, or Slicer + + Raises: + ValueError: if key is not a valid pixel index or slice """ - self.appname = appname - self.frozen = frozen - self.quiet = quiet - self.preserve_dtype = preserve_dtype - self.image_dic = image_dic - if self.quiet: - self.app = otb.Registry.CreateApplicationWithoutLogger(appname) - else: - self.app = otb.Registry.CreateApplication(appname) - self.description = self.app.GetDocLongDescription() - self.output_parameters_keys = self.__get_output_parameters_keys() - if self.output_parameters_keys: - self.output_param = self.output_parameters_keys[0] - - self.parameters = {} - if (args or kwargs): + # Accessing pixel value(s) using Y/X coordinates + if isinstance(key, tuple) and len(key) >= 2: + row, col = key[0], key[1] + if isinstance(row, int) and isinstance(col, int): + if row < 0 or col < 0: + raise ValueError( + f"{self.name} cannot read pixel value at negative coordinates ({row}, {col})" + ) + channels = key[2] if len(key) == 3 else None + return self.get_values_at_coords(row, col, channels) + # Slicing + if not isinstance(key, tuple) or ( + isinstance(key, tuple) and (len(key) < 2 or len(key) > 3) + ): + raise ValueError( + f'"{key}" cannot be interpreted as valid slicing. Slicing should be 2D or 3D.' + ) + if isinstance(key, tuple) and len(key) == 2: + key = key + (slice(None, None, None),) # adding 3rd dimension + return Slicer(self, *key) + + def __repr__(self) -> str: + """Return a string representation with object id. + + This is used as key to store image ref in Operation dicts. + + """ + return f"<pyotb.{self.__class__.__name__} object, id {id(self)}>" + + +class App(OTBObject): + """Wrapper around otb.Application to handle settings and execution. + + Base class that gathers common operations for any OTB application lifetime (settings, exec, export, etc.) + Any app parameter may be passed either using a dict of parameters or keyword argument. + + The first argument can be: + - filepath or OTBObject, the main input parameter name is automatically used + - list of inputs, useful when the user wants to specify the input list `il` + - dictionary of parameters, useful when a key is python-reserved (e.g. `in`, `map`) + Any key except "in" or "map" can also be passed via kwargs, replace "." with "_" e.g `map_epsg_code=4326` + + Args: + appname: name of the OTB application to initialize, e.g. 'BandMath' + *args: can be a filepath, OTB object or a dict or parameters, several dicts will be merged in **kwargs + frozen: freeze OTB app in order avoid blocking during __init___ + quiet: whether to print logs of the OTB app and the default progress bar + name: custom name that will show up in logs, appname will be used if not provided + **kwargs: any OTB application parameter key is accepted except "in" or "map" + + """ + + INPUT_IMAGE_TYPES = [ + otb.ParameterType_InputImage, + otb.ParameterType_InputImageList, + ] + INPUT_PARAM_TYPES = INPUT_IMAGE_TYPES + [ + otb.ParameterType_InputVectorData, + otb.ParameterType_InputVectorDataList, + otb.ParameterType_InputFilename, + otb.ParameterType_InputFilenameList, + ] + OUTPUT_IMAGE_TYPES = [otb.ParameterType_OutputImage] + OUTPUT_PARAM_TYPES = OUTPUT_IMAGE_TYPES + [ + otb.ParameterType_OutputVectorData, + otb.ParameterType_OutputFilename, + ] + INPUT_IMAGES_LIST_TYPES = [ + otb.ParameterType_InputImageList, + otb.ParameterType_InputFilenameList, + ] + INPUT_LIST_TYPES = INPUT_IMAGES_LIST_TYPES + [ + otb.ParameterType_StringList, + otb.ParameterType_ListView, + otb.ParameterType_InputVectorDataList, + otb.ParameterType_Band, + ] + + def __init__( + self, + appname: str, + *args, + frozen: bool = False, + quiet: bool = False, + name: str = "", + **kwargs, + ): + """Common constructor for OTB applications, automatically handles in-memory connections.""" + # Attributes and data structures used by properties + create = ( + otb.Registry.CreateApplicationWithoutLogger + if quiet + else otb.Registry.CreateApplication + ) + self._app = create(appname) + self._name = name or appname + self._exports_dic = {} + self._settings, self._auto_parameters = {}, {} + self._time_start, self._time_end = 0.0, 0.0 + self.data, self.outputs = {}, {} + self.quiet, self.frozen = quiet, frozen + + # Param keys and types + self.parameters_keys = tuple(self.app.GetParametersKeys()) + self._all_param_types = { + key: self.app.GetParameterType(key) for key in self.parameters_keys + } + self._out_param_types = { + key: val + for key, val in self._all_param_types.items() + if val in self.OUTPUT_PARAM_TYPES + } + self._key_choices = { + key: [f"{key}.{choice}" for choice in self.app.GetChoiceKeys(key)] + for key in self.parameters_keys + if self.app.GetParameterType(key) == otb.ParameterType_Choice + } + + # Init, execute and write (auto flush only when output param was provided) + if args or kwargs: self.set_parameters(*args, **kwargs) + # Create Output image objects + for key in ( + key + for key, param in self._out_param_types.items() + if param == otb.ParameterType_OutputImage + ): + self.outputs[key] = Output(self, key, self._settings.get(key)) + if not self.frozen: self.execute() + if any(key in self._settings for key in self._out_param_types): + self.flush() + else: + self.__sync_parameters() # since not called during execute() + + @property + def name(self) -> str: + """Returns appname by default, or a custom name if passed during App init.""" + return self._name + + @property + def app(self) -> otb.Application: + """Reference to this app otb.Application instance.""" + return self._app + + @property + def parameters(self): + """Return used application parameters: automatic values or set by user.""" + return {**self._auto_parameters, **self._settings} + + @property + def exports_dic(self) -> dict[str, dict]: + """Reference to an internal dict object that contains numpy array exports.""" + return self._exports_dic + + def __is_one_of_types(self, key: str, param_types: list[int]) -> bool: + """Helper to check the type of a parameter.""" + if key not in self._all_param_types: + raise KeyError(f"key {key} not found in the application parameters types") + return self._all_param_types[key] in param_types + + def __is_multi_output(self): + """Check if app has multiple outputs to ensure re-execution during write().""" + return len(self.outputs) > 1 + + def is_input(self, key: str) -> bool: + """Returns True if the parameter key is an input.""" + return self.__is_one_of_types(key=key, param_types=self.INPUT_PARAM_TYPES) + + def is_output(self, key: str) -> bool: + """Returns True if the parameter key is an output.""" + return self.__is_one_of_types(key=key, param_types=self.OUTPUT_PARAM_TYPES) + + def is_key_list(self, key: str) -> bool: + """Check if a parameter key is an input parameter list.""" + return self.app.GetParameterType(key) in self.INPUT_LIST_TYPES + + def is_key_images_list(self, key: str) -> bool: + """Check if a parameter key is an input parameter image list.""" + return self.app.GetParameterType(key) in self.INPUT_IMAGES_LIST_TYPES + + def get_first_key(self, param_types: list[int]) -> str: + """Get the first param key for specific file types, try each list in args.""" + for param_type in param_types: + # Return the first key, from the alphabetically sorted keys of the + # application, which has the parameter type matching param_type. + for key, value in sorted(self._all_param_types.items()): + if value == param_type: + return key + raise TypeError( + f"{self.name}: could not find any key matching the provided types" + ) + + @property + def input_key(self) -> str: + """Get the name of first input parameter, raster > vector > file.""" + return self.get_first_key(self.INPUT_PARAM_TYPES) + + @property + def input_image_key(self) -> str: + """Name of the first input image parameter.""" + return self.get_first_key(self.INPUT_IMAGE_TYPES) + + @property + def output_key(self) -> str: + """Name of the first output parameter, raster > vector > file.""" + return self.get_first_key(self.OUTPUT_PARAM_TYPES) + + @property + def output_image_key(self) -> str: + """Get the name of first output image parameter.""" + return self.get_first_key(self.OUTPUT_IMAGE_TYPES) + + @property + def elapsed_time(self) -> float: + """Get elapsed time between app init and end of exec or file writing.""" + return self._time_end - self._time_start def set_parameters(self, *args, **kwargs): - """Set some parameters of the app. + """Set parameters, using the right OTB API function depending on the key and type. + Parameters with dots may be passed as keyword arguments using "_", e.g. map_epsg_code=4326. + Additional checks are done for input and output (in-memory objects, remote filepaths, etc.). When useful, e.g. for images list, this function appends the parameters instead of overwriting them. Handles any parameters, i.e. in-memory & filepaths Args: - *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved - (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit") - - string, App or Output, useful when the user implicitly wants to set the param "in" - - list, useful when the user implicitly wants to set the param "il" - **kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif' + *args: any input OTBObject, filepath or images list, or a dict of parameters + **kwargs: app parameters, with "_" instead of dots e.g. io_in="image.tif" Raises: - Exception: when the setting of a parameter failed + KeyError: when the parameter name wasn't recognized + RuntimeError: failed to set parameter value """ parameters = kwargs parameters.update(self.__parse_args(args)) # Going through all arguments - for param, obj in parameters.items(): - if param not in self.app.GetParametersKeys(): - raise Exception(f"{self.name}: parameter '{param}' was not recognized. " - f"Available keys are {self.app.GetParametersKeys()}") + for key, obj in parameters.items(): + if "_" in key: + key = key.replace("_", ".") + if key not in self.parameters_keys: + raise KeyError( + f"{self.name}: parameter '{key}' was not recognized." + f" Available keys are {self.parameters_keys}" + ) # When the parameter expects a list, if needed, change the value to list - if self.__is_key_list(param) and not isinstance(obj, (list, tuple)): - parameters[param] = [obj] + if self.is_key_list(key) and not isinstance(obj, (list, tuple)): obj = [obj] - logger.warning('%s: argument for parameter "%s" was converted to list', self.name, param) + logger.info( + '%s: argument for parameter "%s" was converted to list', + self.name, + key, + ) try: - # This is when we actually call self.app.SetParameter* - self.__set_param(param, obj) + if self.is_input(key): + obj = self.__check_input_param(obj) + elif self.is_output(key): + obj = self.__check_output_param(obj) + self.__set_param(key, obj) except (RuntimeError, TypeError, ValueError, KeyError) as e: - raise Exception(f"{self.name}: something went wrong before execution " - f"(while setting parameter '{param}' to '{obj}')") from e - # Update _parameters using values from OtbApplication object - otb_params = self.app.GetParameters().items() - otb_params = {k: str(v) if isinstance(v, otb.ApplicationProxy) else v for k, v in otb_params} - self.parameters.update({**parameters, **otb_params}) - # Update output images pixel types - if self.preserve_dtype: - self.__propagate_pixel_type() + raise RuntimeError( + f"{self.name}: error before execution," + f" while setting '{key}' to '{obj}': {e})" + ) from e + # Save / update setting value and update the Output object initialized in __init__ without a filepath + self._settings[key] = obj + if key in self.outputs: + self.outputs[key].filepath = obj + if key in self._auto_parameters: + del self._auto_parameters[key] + + def propagate_dtype(self, target_key: str = None, dtype: int = None): + """Propagate a pixel type from main input to every outputs, or to a target output key only. + + With multiple inputs (if dtype is not provided), the type of the first input is considered. + With multiple outputs (if target_key is not provided), all outputs will be converted to the same pixel type. + + Args: + target_key: output param key to change pixel type + dtype: data type to use + + """ + if not dtype: + param = self._settings.get(self.input_image_key) + if not param: + logger.warning( + "%s: could not propagate pixel type from inputs to output", + self.name, + ) + return + if isinstance(param, (list, tuple)): + param = param[0] # first image in "il" + try: + dtype = get_pixel_type(param) + except (TypeError, RuntimeError): + logger.warning( + '%s: unable to identify pixel type of key "%s"', self.name, param + ) + return + if target_key: + keys = [target_key] + else: + keys = [ + k + for k, v in self._out_param_types.items() + if v == otb.ParameterType_OutputImage + ] + for key in keys: + self.app.SetParameterOutputImagePixelType(key, dtype) def execute(self): """Execute and write to disk if any output parameter has been set during init.""" logger.debug("%s: run execute() with parameters=%s", self.name, self.parameters) + self._time_start = perf_counter() try: self.app.Execute() except (RuntimeError, FileNotFoundError) as e: - raise Exception(f'{self.name}: error during during app execution') from e + raise RuntimeError( + f"{self.name}: error during during app execution ({e}" + ) from e self.frozen = False + self._time_end = perf_counter() logger.debug("%s: execution ended", self.name) - if self.__has_output_param_key(): - logger.debug('%s: flushing data to disk', self.name) - self.app.WriteOutput() - self.__save_objects() + self.__sync_parameters() + + def flush(self): + """Flush data to disk, this is when WriteOutput is actually called.""" + logger.debug("%s: flushing data to disk", self.name) + self.app.WriteOutput() + self._time_end = perf_counter() + + @deprecated_alias(filename_extension="ext_fname") + def write( + self, + path: str | Path | dict[str, str] = None, + pixel_type: dict[str, str] | str = None, + preserve_dtype: bool = False, + ext_fname: dict[str, str] | str = None, + **kwargs, + ) -> bool: + """Set output pixel type and write the output raster files. + + The first argument is expected to be: + - filepath, useful when there is only one output, e.g. 'output.tif' + - dictionary containing output filepath + - None if output file was passed during App init + + In case of multiple outputs, pixel_type may also be a dictionary with parameter names as keys. + Accepted pixel types : uint8, uint16, uint32, int16, int32, float, double, cint16, cint32, cfloat, cdouble - def find_output(self): - """Find output files on disk using path found in parameters. + Args: + path: output filepath or dict of filepath with param keys + pixel_type: pixel type string representation + preserve_dtype: propagate main input pixel type to outputs, in case pixel_type is None + ext_fname: an OTB extended filename, will be applied to every output (but won't overwrite existing keys in output filepath) + **kwargs: keyword arguments e.g. out='output.tif' or io_out='output.tif' Returns: - list of files found on disk - - """ - files = [] - missing = [] - outputs = [p for p in self.output_parameters_keys if p in self.parameters] - for param in outputs: - filename = self.parameters[param] - # Remove filename extension - if '?' in filename: - filename = filename.split('?')[0] - path = Path(filename) - if path.exists(): - files.append(str(path.absolute())) - else: - missing.append(str(path.absolute())) - if missing: - missing = tuple(missing) - for filename in missing: - logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename) - - return files - - # Private functions - def __get_output_parameters_keys(self): - """Get raster output parameter keys. + True if all files are found on disk - Returns: - output parameters keys """ - return [param for param in self.app.GetParametersKeys() - if self.app.GetParameterType(param) == otb.ParameterType_OutputImage] - - def __has_output_param_key(self): - """Check if App has any output parameter key.""" - if not self.output_param: - return True # apps like ReadImageInfo with no filetype output param still needs to WriteOutput - types = (otb.ParameterType_OutputFilename, otb.ParameterType_OutputImage, otb.ParameterType_OutputVectorData) - outfile_params = [param for param in self.app.GetParametersKeys() if self.app.GetParameterType(param) in types] - return any(key in self.parameters for key in outfile_params) - - @staticmethod - def __parse_args(args): + # Gather all input arguments in kwargs dict + if isinstance(path, dict): + kwargs.update(path) + elif isinstance(path, str) and kwargs: + logger.warning( + '%s: keyword arguments specified, ignoring argument "%s"', + self.name, + path, + ) + elif isinstance(path, (str, Path)) and self.output_key: + kwargs[self.output_key] = str(path) + elif not path and self.output_key in self.parameters: + kwargs[self.output_key] = self.parameters[self.output_key] + elif path is not None: + raise TypeError(f"{self.name}: unsupported filepath type ({type(path)})") + if not (kwargs or any(k in self._settings for k in self._out_param_types)): + raise KeyError( + f"{self.name}: at least one filepath is required, if not provided during App init" + ) + parameters = kwargs.copy() + + # Append filename extension to filenames + if ext_fname: + if not isinstance(ext_fname, (dict, str)): + raise ValueError("Extended filename must be a str or a dict") + + def _str2dict(ext_str): + """Function that converts str to dict.""" + splits = [pair.split("=") for pair in ext_str.split("&")] + return dict(split for split in splits if len(split) == 2) + + if isinstance(ext_fname, str): + ext_fname = _str2dict(ext_fname) + logger.debug("%s: extended filename for all outputs:", self.name) + for key, ext in ext_fname.items(): + logger.debug("%s: %s", key, ext) + + for key, filepath in kwargs.items(): + if self._out_param_types[key] == otb.ParameterType_OutputImage: + new_ext_fname = ext_fname.copy() + # Grab already set extended filename key/values + if "?&" in filepath: + filepath, already_set_ext = filepath.split("?&", 1) + # Extensions in filepath prevail over `new_ext_fname` + new_ext_fname.update(_str2dict(already_set_ext)) + # tyransform dict to str + ext_fname_str = "&".join( + [f"{key}={value}" for key, value in new_ext_fname.items()] + ) + parameters[key] = f"{filepath}?&{ext_fname_str}" + + # Manage output pixel types + data_types = {} + if pixel_type: + if isinstance(pixel_type, str): + dtype = parse_pixel_type(pixel_type) + type_name = self.app.ConvertPixelTypeToNumpy(dtype) + logger.debug( + '%s: output(s) will be written with type "%s"', self.name, type_name + ) + for key in parameters: + if self._out_param_types[key] == otb.ParameterType_OutputImage: + data_types[key] = dtype + elif isinstance(pixel_type, dict): + data_types = { + key: parse_pixel_type(dtype) for key, dtype in pixel_type.items() + } + elif preserve_dtype: + self.propagate_dtype() + + # Set parameters and flush to disk + for key, filepath in parameters.items(): + if Path(filepath.split("?")[0]).exists(): + logger.warning("%s: overwriting file %s", self.name, filepath) + if key in data_types: + self.propagate_dtype(key, data_types[key]) + self.set_parameters({key: filepath}) + # TODO: drop multioutput special case when fixed on the OTB side. See discussion in MR !102 + if self.frozen or self.__is_multi_output(): + self.execute() + self.flush() + if not parameters: + return True + + # Search and log missing files + files, missing = [], [] + for key, filepath in parameters.items(): + if not filepath.startswith("/vsi"): + filepath = Path(filepath.split("?")[0]) + dest = files if filepath.exists() else missing + dest.append(str(filepath.absolute())) + for filename in missing: + logger.error( + "%s: execution seems to have failed, %s does not exist", + self.name, + filename, + ) + return bool(files) and not missing + + def __parse_args(self, args: list[str | OTBObject | dict | list]) -> dict[str, Any]: """Gather all input arguments in kwargs dict. + Args: + args: the list of arguments passed to set_parameters (__init__ *args) + Returns: a dictionary with the right keyword depending on the object @@ -707,238 +956,255 @@ class App(otbObject): for arg in args: if isinstance(arg, dict): kwargs.update(arg) - elif isinstance(arg, (str, otbObject)): - kwargs.update({'in': arg}) - elif isinstance(arg, list): - kwargs.update({'il': arg}) + elif ( + isinstance(arg, (str, OTBObject)) + or isinstance(arg, list) + and self.is_key_list(self.input_key) + ): + kwargs.update({self.input_key: arg}) return kwargs - def __set_param(self, param, obj): + def __check_input_param( + self, obj: list | tuple | OTBObject | str | Path + ) -> list | OTBObject | str: + """Check the type and value of an input parameter, add vsi prefixes if needed.""" + if isinstance(obj, (list, tuple)): + return [self.__check_input_param(o) for o in obj] + if isinstance(obj, OTBObject): + return obj + if isinstance(obj, Path): + obj = str(obj) + if isinstance(obj, str): + if not obj.startswith("/vsi"): + # Remote file. TODO: add support for S3 / GS / AZ + if obj.startswith(("https://", "http://", "ftp://")): + obj = "/vsicurl/" + obj + prefixes = { + ".tar": "vsitar", + ".tar.gz": "vsitar", + ".tgz": "vsitar", + ".gz": "vsigzip", + ".7z": "vsi7z", + ".zip": "vsizip", + ".rar": "vsirar", + } + expr = r"(.*?)(\.7z|\.zip|\.rar|\.tar\.gz|\.tgz|\.tar|\.gz)(.*)" + parts = re.match(expr, obj) + if parts: + file, ext = parts.group(1), parts.group(2) + if not Path(file + ext).is_dir(): + obj = f"/{prefixes[ext]}/{obj}" + return obj + raise TypeError(f"{self.name}: wrong input parameter type ({type(obj)})") + + def __check_output_param(self, obj: list | tuple | str | Path) -> list | str: + """Check the type and value of an output parameter.""" + if isinstance(obj, (list, tuple)): + return [self.__check_output_param(o) for o in obj] + if isinstance(obj, Path): + obj = str(obj) + if isinstance(obj, str): + return obj + raise TypeError(f"{self.name}: wrong output parameter type ({type(obj)})") + + def __set_param( + self, key: str, obj: str | float | list | tuple | OTBObject | otb.Application + ): """Set one parameter, decide which otb.Application method to use depending on target object.""" - if obj is not None: - # Single-parameter cases - if isinstance(obj, otbObject): - self.app.ConnectImage(param, obj.app, obj.output_param) - elif isinstance(obj, otb.Application): # this is for backward comp with plain OTB - outparamkey = [param for param in obj.GetParametersKeys() - if obj.GetParameterType(param) == otb.ParameterType_OutputImage][0] - self.app.ConnectImage(param, obj, outparamkey) - elif param == 'ram': # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf gitlab OTB issue 2200 - self.app.SetParameterInt('ram', int(obj)) - elif not isinstance(obj, list): # any other parameters (str, int...) - self.app.SetParameterValue(param, obj) - # Images list - elif self.__is_key_images_list(param): - # To enable possible in-memory connections, we go through the list and set the parameters one by one - for inp in obj: - if isinstance(inp, otbObject): - self.app.ConnectImage(param, inp.app, inp.output_param) - elif isinstance(inp, otb.Application): # this is for backward comp with plain OTB - outparamkey = [param for param in inp.GetParametersKeys() if - inp.GetParameterType(param) == otb.ParameterType_OutputImage][0] - self.app.ConnectImage(param, inp, outparamkey) - else: # here `input` should be an image filepath - # Append `input` to the list, do not overwrite any previously set element of the image list - self.app.AddParameterStringList(param, inp) - # List of any other types (str, int...) - else: - self.app.SetParameterValue(param, obj) - - def __propagate_pixel_type(self): - """Propagate the pixel type from inputs to output. - - For several inputs, or with an image list, the type of the first input is considered. - If several outputs, all outputs will have the same type. - - """ - pixel_type = None - for key, param in self.parameters.items(): - if self.__is_key_input_image(key): - if not param: - continue - if isinstance(param, list): - param = param[0] # first image in "il" - try: - pixel_type = get_pixel_type(param) - type_name = self.app.ConvertPixelTypeToNumpy(pixel_type) - logger.debug('%s: output(s) will be written with type "%s"', self.name, type_name) - for out_key in self.output_parameters_keys: - self.app.SetParameterOutputImagePixelType(out_key, pixel_type) - return - except TypeError: - pass - - logger.warning("%s: could not propagate pixel type from inputs to output, no valid input found", self.name) + if obj is None or (isinstance(obj, (list, tuple)) and not obj): + self.app.ClearValue(key) + return + # Single-parameter cases + if isinstance(obj, OTBObject): + self.app.ConnectImage(key, obj.app, obj.output_image_key) + elif isinstance(obj, otb.Application): + self.app.ConnectImage(key, obj, get_out_images_param_keys(obj)[0]) + elif key == "ram": + # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf OTB issue 2200 + self.app.SetParameterInt("ram", int(obj)) + # Any other parameters (str, int...) + elif not isinstance(obj, (list, tuple)): + self.app.SetParameterValue(key, obj) + # Images list + elif self.is_key_images_list(key): + for inp in obj: + if isinstance(inp, OTBObject): + self.app.ConnectImage(key, inp.app, inp.output_image_key) + elif isinstance(inp, otb.Application): + self.app.ConnectImage(key, obj, get_out_images_param_keys(inp)[0]) + # Here inp is either str or Path, already checked by __check_*_param + else: + # Append it to the list, do not overwrite any previously set element of the image list + self.app.AddParameterStringList(key, inp) + # List of any other types (str, int...) + elif self.is_key_list(key): + self.app.SetParameterValue(key, obj) + else: + raise TypeError( + f"{self.name}: wrong parameter type ({type(obj)}) for '{key}'" + ) - def __save_objects(self): - """Saving app parameters and outputs as attributes, so that they can be accessed with `obj.key`. + def __sync_parameters(self): + """Save app parameters in _auto_parameters or data dict. - This is useful when the key contains reserved characters such as a point eg "io.out" + This is always called during init or after execution, to ensure the + parameters property of the App is in sync with the otb.Application instance. """ - for key in self.app.GetParametersKeys(): - if key == 'parameters': # skip forbidden attribute since it is already used by the App class + skip = [ + k for k in self.parameters_keys if k.split(".")[-1] in ("ram", "default") + ] + # Prune unused choices child params + for key in self._key_choices: + choices = self._key_choices[key].copy() + choices.remove(f"{key}.{self.app.GetParameterValue(key)}") + skip.extend( + [k for k in self.parameters_keys if k.startswith(tuple(choices))] + ) + + self._auto_parameters.clear() + for key in self.parameters_keys: + if key in skip or key in self._settings or not self.app.HasValue(key): continue - value = None - if key in self.output_parameters_keys: # raster outputs - value = Output(self, key) - elif key in self.parameters: # user or default app parameters - value = self.parameters[key] - else: # any other app attribute (e.g. ReadImageInfo results) + value = self.app.GetParameterValue(key) + if isinstance(value, otb.ApplicationProxy): try: - value = self.app.GetParameterValue(key) + value = str(value) except RuntimeError: - pass # this is when there is no value for key - if value is not None: - setattr(self, key, value) - - def __is_key_input_image(self, key): - """Check if a key of the App is an input parameter image list.""" - return self.app.GetParameterType(key) in (otb.ParameterType_InputImage, otb.ParameterType_InputImageList) - - def __is_key_list(self, key): - """Check if a key of the App is an input parameter list.""" - return self.app.GetParameterType(key) in (otb.ParameterType_InputImageList, otb.ParameterType_StringList, - otb.ParameterType_InputFilenameList, otb.ParameterType_ListView, - otb.ParameterType_InputVectorDataList) - - def __is_key_images_list(self, key): - """Check if a key of the App is an input parameter image list.""" - return self.app.GetParameterType(key) in (otb.ParameterType_InputImageList, otb.ParameterType_InputFilenameList) - - # Special methods - def __str__(self): - """Return a nice string representation with object id.""" - return f'<pyotb.App {self.appname} object id {id(self)}>' + continue + # Keep False or 0 values, but make sure to skip empty collections or str + if hasattr(value, "__iter__") and not value: + continue + # Here we should use AND self.app.IsParameterEnabled(key) but it's broken + if self.app.GetParameterRole(key) == 0 and ( + self.app.HasAutomaticValue(key) or self.app.IsParameterEnabled(key) + ): + self._auto_parameters[key] = value + # Save static output data (ReadImageInfo, ComputeImageStatistics, etc.) + elif self.app.GetParameterRole(key) == 1: + if isinstance(value, str): + try: + value = literal_eval(value) + except (ValueError, SyntaxError): + pass + self.data[key] = value + + # Special functions + def __getitem__(self, key: str | tuple) -> Any | list[float] | float | Slicer: + """This function is called when we use App()[...]. + + We allow to return attr if key is a parameter, or call OTBObject __getitem__ for pixel values or Slicer + """ + if isinstance(key, tuple): + return super().__getitem__(key) # to read pixel values, or slice + if isinstance(key, str): + if key in self.data: + return self.data[key] + if key in self.outputs: + return self.outputs[key] + if key in self.parameters: + return self.parameters[key] + raise KeyError(f"{self.name}: unknown or undefined parameter '{key}'") + raise TypeError( + f"{self.name}: cannot access object item or slice using {type(key)} object" + ) class Slicer(App): - """Slicer objects i.e. when we call something like raster[:, :, 2] from Python.""" + """Slicer objects, automatically created when using slicing e.g. app[:, :, 2]. - def __init__(self, x, rows, cols, channels): - """Create a slicer object, that can be used directly for writing or inside a BandMath. + Can be used to select a subset of pixel and / or bands in the image. + This is a shortcut to an ExtractROI app that can be written to disk or used in pipelines. - It contains : - - an ExtractROI app that handles extracting bands and ROI and can be written to disk or used in pipelines - - in case the user only wants to extract one band, an expression such as "im1b#" + Args: + obj: input + rows: slice along Y / Latitude axis + cols: slice along X / Longitude axis + channels: bands to extract - Args: - x: input - rows: slice along Y / Latitude axis - cols: slice along X / Longitude axis - channels: channels, can be slicing, list or int + Raises: + TypeError: if channels param isn't slice, list or int - """ - # Initialize the app that will be used for writing the slicer - self.name = 'Slicer' + """ + + def __init__( + self, + obj: OTBObject, + rows: slice, + cols: slice, + channels: slice | list[int] | int, + ): + """Create a slicer object, that can be used directly for writing or inside a BandMath.""" + super().__init__( + "ExtractROI", + obj, + mode="extent", + quiet=True, + frozen=True, + name=f"Slicer from {obj.name}", + ) + self.rows, self.cols = rows, cols + parameters = {} - self.output_parameter_key = 'out' - parameters = {'in': x, 'mode': 'extent'} - super().__init__('ExtractROI', parameters, preserve_dtype=True, frozen=True) # Channel slicing if channels != slice(None, None, None): - # Trigger source app execution if needed - nb_channels = get_nbchannels(x) + nb_channels = get_nbchannels(obj) self.app.Execute() # this is needed by ExtractROI for setting the `cl` parameter - # if needed, converting int to list if isinstance(channels, int): channels = [channels] - # if needed, converting slice to list elif isinstance(channels, slice): - channels_start = channels.start if channels.start is not None else 0 - channels_start = channels_start if channels_start >= 0 else nb_channels + channels_start - channels_end = channels.stop if channels.stop is not None else nb_channels - channels_end = channels_end if channels_end >= 0 else nb_channels + channels_end - channels_step = channels.step if channels.step is not None else 1 - channels = range(channels_start, channels_end, channels_step) + channels = self.channels_list_from_slice(channels) elif isinstance(channels, tuple): channels = list(channels) elif not isinstance(channels, list): - raise ValueError(f'Invalid type for channels, should be int, slice or list of bands. : {channels}') - + raise TypeError( + f"Invalid type for channels ({type(channels)})." + f" Should be int, slice or list of bands." + ) # Change the potential negative index values to reverse index channels = [c if c >= 0 else nb_channels + c for c in channels] - parameters.update({'cl': [f'Channel{i + 1}' for i in channels]}) + parameters.update({"cl": [f"Channel{i + 1}" for i in channels]}) # Spatial slicing spatial_slicing = False - # TODO: handle PixelValue app so that accessing value is possible, e.g. raster[120, 200, 0] - # TODO TBD: handle the step value in the slice so that NN undersampling is possible ? e.g. raster[::2, ::2] if rows.start is not None: - parameters.update({'mode.extent.uly': rows.start}) + parameters.update({"mode.extent.uly": rows.start}) spatial_slicing = True if rows.stop is not None and rows.stop != -1: - parameters.update( - {'mode.extent.lry': rows.stop - 1}) # subtract 1 to be compliant with python convention + # Subtract 1 to respect python convention + parameters.update({"mode.extent.lry": rows.stop - 1}) spatial_slicing = True if cols.start is not None: - parameters.update({'mode.extent.ulx': cols.start}) + parameters.update({"mode.extent.ulx": cols.start}) spatial_slicing = True if cols.stop is not None and cols.stop != -1: - parameters.update( - {'mode.extent.lrx': cols.stop - 1}) # subtract 1 to be compliant with python convention + # Subtract 1 to respect python convention + parameters.update({"mode.extent.lrx": cols.stop - 1}) spatial_slicing = True - # Execute app - self.set_parameters(**parameters) - self.execute() - - # These are some attributes when the user simply wants to extract *one* band to be used in an Operation + # When the user simply wants to extract *one* band to be used in an Operation if not spatial_slicing and isinstance(channels, list) and len(channels) == 1: - self.one_band_sliced = channels[0] + 1 # OTB convention: channels start at 1 - self.input = x - - -class Input(App): - """Class for transforming a filepath to pyOTB object.""" - - def __init__(self, filepath): - """Constructor for an Input object. - - Args: - filepath: raster file path - - """ - self.filepath = filepath - super().__init__('ExtractROI', {'in': self.filepath}, preserve_dtype=True) - - def __str__(self): - """Return a nice string representation with input file path.""" - return f'<pyotb.Input object from {self.filepath}>' + # OTB convention: channels start at 1 + self.one_band_sliced = channels[0] + 1 + self.input = obj - -class Output(otbObject): - """Class for output of an app.""" - - def __init__(self, app, output_parameter_key): - """Constructor for an Output object. - - Args: - app: The pyotb App - output_parameter_key: Output parameter key - - """ - # Keeping the OTB app and the pyotb app - self.pyotb_app, self.app = app, app.app - self.parameters = self.pyotb_app.parameters - self.output_param = output_parameter_key - self.name = f'Output {output_parameter_key} from {self.app.GetName()}' - - def summarize(self): - """Return the summary of the pipeline that generates the Output object. - - Returns: - Nested dictionary summarizing the pipeline that generates the Output object. - - """ - return self.pyotb_app.summarize() - - def __str__(self): - """Return a nice string representation with object id.""" - return f'<pyotb.Output {self.app.GetName()} object, id {id(self)}>' + # Execute app + self.set_parameters(parameters) + self.propagate_dtype() + self.execute() class Operation(App): """Class for arithmetic/math operations done in Python. + Given some inputs and an operator, this object enables to python operator to a BandMath operation. + Operations generally involve 2 inputs (+, -...). It can have only 1 input for `abs` operator. + It can have 3 inputs for the ternary operator `cond ? x : y`. + + Args: + operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ? + *inputs: operands of the expression to build + nb_bands: optionally specify the output nb of bands - used only internally by pyotb.where + name: override the default Operation name + Example: Consider the python expression (input1 + 2 * input2) > 0. This class enables to create a BandMathX app, with expression such as (im2 + 2 * im1) > 0 ? 1 : 0 @@ -957,120 +1223,130 @@ class Operation(App): """ - def __init__(self, operator, *inputs, nb_bands=None): - """Given some inputs and an operator, this function enables to transform this into an OTB application. - - Operations generally involve 2 inputs (+, -...). It can have only 1 input for `abs` operator. - It can have 3 inputs for the ternary operator `cond ? x : y`. - - Args: - operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ? - *inputs: inputs. Can be App, Output, Input, Operation, Slicer, filepath, int or float - nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where - - """ + def __init__(self, operator: str, *inputs, nb_bands: int = None, name: str = None): + """Operation constructor, one part of the logic is handled by App.__create_operator.""" self.operator = operator - # We first create a 'fake' expression. E.g for the operation `input1 + input2` , we create a fake expression - # that is like "str(input1) + str(input2)" + # We first create a 'fake' expression. E.g for the operation `input1 + input2` + # we create a fake expression like "str(input1) + str(input2)" self.inputs = [] self.nb_channels = {} self.fake_exp_bands = [] - self.logical_fake_exp_bands = [] - - self.create_fake_exp(operator, inputs, nb_bands=nb_bands) - + self.build_fake_expressions(operator, inputs, nb_bands=nb_bands) # Transforming images to the adequate im#, e.g. `input1` to "im1" - # creating a dictionary that is like {str(input1): 'im1', 'image2.tif': 'im2', ...}. + # using a dictionary : {str(input1): 'im1', 'image2.tif': 'im2', ...}. # NB: the keys of the dictionary are strings-only, instead of 'complex' objects, to enable easy serialization self.im_dic = {} self.im_count = 1 - mapping_str_to_input = {} # to be able to retrieve the real python object from its string representation + # To be able to retrieve the real python object from its string representation + map_repr_to_input = {} for inp in self.inputs: if not isinstance(inp, (int, float)): if str(inp) not in self.im_dic: - self.im_dic[str(inp)] = f'im{self.im_count}' - mapping_str_to_input[str(inp)] = inp + self.im_dic[repr(inp)] = f"im{self.im_count}" + map_repr_to_input[repr(inp)] = inp self.im_count += 1 + # Getting unique image inputs, in the order im1, im2, im3 ... + self.unique_inputs = [ + map_repr_to_input[id_str] + for id_str in sorted(self.im_dic, key=self.im_dic.get) + ] + self.exp_bands, self.exp = self.get_real_exp(self.fake_exp_bands) + appname = "BandMath" if len(self.exp_bands) == 1 else "BandMathX" + name = f'Operation exp="{self.exp}"' + super().__init__( + appname, il=self.unique_inputs, exp=self.exp, quiet=True, name=name + ) - # getting unique image inputs, in the order im1, im2, im3 ... - self.unique_inputs = [mapping_str_to_input[str_input] for str_input in sorted(self.im_dic, key=self.im_dic.get)] - self.output_param = 'out' + def get_nb_bands(self, inputs: list[OTBObject | str | float]) -> int: + """Guess the number of bands of the output image, from the inputs. - # Computing the BandMath or BandMathX app - self.exp_bands, self.exp = self.get_real_exp(self.fake_exp_bands) - self.name = f'Operation exp="{self.exp}"' + Args: + inputs: the Operation operands - appname = 'BandMath' if len(self.exp_bands) == 1 else 'BandMathX' - super().__init__(appname, il=self.unique_inputs, exp=self.exp) + Raises: + ValueError: if all inputs don't have the same number of bands - def create_fake_exp(self, operator, inputs, nb_bands=None): - """Create a 'fake' expression. + """ + if any( + isinstance(inp, Slicer) and hasattr(inp, "one_band_sliced") + for inp in inputs + ): + return 1 + # Check that all inputs have the same band count + nb_bands_list = [ + get_nbchannels(inp) for inp in inputs if not isinstance(inp, (float, int)) + ] + all_same = all(x == nb_bands_list[0] for x in nb_bands_list) + if len(nb_bands_list) > 1 and not all_same: + raise ValueError("All images do not have the same number of bands") + return nb_bands_list[0] + + def build_fake_expressions( + self, + operator: str, + inputs: list[OTBObject | str | float], + nb_bands: int = None, + ): + """Create a list of 'fake' expressions, one for each band. E.g for the operation input1 + input2, we create a fake expression that is like "str(input1) + str(input2)" Args: - operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ? - inputs: inputs. Can be App, Output, Input, Operation, Slicer, filepath, int or float + operator: one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ? + inputs: inputs. Can be OTBObject, filepath, int or float nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where + Raises: + ValueError: if all inputs don't have the same number of bands + """ self.inputs.clear() self.nb_channels.clear() - logger.debug("%s, %s", operator, inputs) - # this is when we use the ternary operator with `pyotb.where` function. The output nb of bands is already known - if operator == '?' and nb_bands: + # When we use the ternary operator with `pyotb.where` function, the output nb of bands is already known + if operator == "?" and nb_bands: pass # For any other operations, the output number of bands is the same as inputs else: - if any(isinstance(inp, Slicer) and hasattr(inp, 'one_band_sliced') for inp in inputs): - nb_bands = 1 - else: - nb_bands_list = [get_nbchannels(inp) for inp in inputs if not isinstance(inp, (float, int))] - # check that all inputs have the same nb of bands - if len(nb_bands_list) > 1: - if not all(x == nb_bands_list[0] for x in nb_bands_list): - raise Exception('All images do not have the same number of bands') - nb_bands = nb_bands_list[0] - + nb_bands = self.get_nb_bands(inputs) # Create a list of fake expressions, each item of the list corresponding to one band self.fake_exp_bands.clear() for i, band in enumerate(range(1, nb_bands + 1)): - fake_exps = [] + expressions = [] for k, inp in enumerate(inputs): - # Generating the fake expression of the current input + # Generating the fake expression of the current input, # this is a special case for the condition of the ternary operator `cond ? x : y` if len(inputs) == 3 and k == 0: - # when cond is monoband whereas the result is multiband, we expand the cond to multiband - if nb_bands != inp.shape[2]: - cond_band = 1 - else: - cond_band = band - fake_exp, corresponding_inputs, nb_channels = self.create_one_input_fake_exp(inp, cond_band, - keep_logical=True) - # any other input + # When cond is monoband whereas the result is multiband, we expand the cond to multiband + cond_band = 1 if nb_bands != inp.shape[2] else band + fake_exp, corresp_inputs, nb_channels = self.make_fake_exp( + inp, cond_band, keep_logical=True + ) else: - fake_exp, corresponding_inputs, nb_channels = self.create_one_input_fake_exp(inp, band, - keep_logical=False) - fake_exps.append(fake_exp) + # Any other input + fake_exp, corresp_inputs, nb_channels = self.make_fake_exp( + inp, band, keep_logical=False + ) + expressions.append(fake_exp) # Reference the inputs and nb of channels (only on first pass in the loop to avoid duplicates) - if i == 0 and corresponding_inputs and nb_channels: - self.inputs.extend(corresponding_inputs) + if i == 0 and corresp_inputs and nb_channels: + self.inputs.extend(corresp_inputs) self.nb_channels.update(nb_channels) # Generating the fake expression of the whole operation - if len(inputs) == 1: # this is only for 'abs' - fake_exp = f'({operator}({fake_exps[0]}))' + if len(inputs) == 1: + # This is only for 'abs()' + fake_exp = f"({operator}({expressions[0]}))" elif len(inputs) == 2: # We create here the "fake" expression. For example, for a BandMathX expression such as '2 * im1 + im2', # the false expression stores the expression 2 * str(input1) + str(input2) - fake_exp = f'({fake_exps[0]} {operator} {fake_exps[1]})' - elif len(inputs) == 3 and operator == '?': # this is only for ternary expression - fake_exp = f'({fake_exps[0]} ? {fake_exps[1]} : {fake_exps[2]})' - + fake_exp = f"({expressions[0]} {operator} {expressions[1]})" + elif len(inputs) == 3 and operator == "?": + # This is only for ternary expression + fake_exp = f"({expressions[0]} ? {expressions[1]} : {expressions[2]})" self.fake_exp_bands.append(fake_exp) - def get_real_exp(self, fake_exp_bands): + def get_real_exp(self, fake_exp_bands: str) -> tuple[list[str], str]: """Generates the BandMathX expression. Args: @@ -1086,28 +1362,27 @@ class Operation(App): for one_band_fake_exp in fake_exp_bands: one_band_exp = one_band_fake_exp for inp in self.inputs: - # replace the name of in-memory object (e.g. '<pyotb.App object>b1' by 'im1b1') - one_band_exp = one_band_exp.replace(str(inp), self.im_dic[str(inp)]) + # Replace the name of in-memory object (e.g. '<pyotb.App object>b1' by 'im1b1') + one_band_exp = one_band_exp.replace(repr(inp), self.im_dic[repr(inp)]) exp_bands.append(one_band_exp) - # Form the final expression (e.g. 'im1b1 + 1; im1b2 + 1') - exp = ';'.join(exp_bands) - - return exp_bands, exp + return exp_bands, ";".join(exp_bands) @staticmethod - def create_one_input_fake_exp(x, band, keep_logical=False): - """This an internal function, only to be used by `create_fake_exp`. + def make_fake_exp( + x: OTBObject | str, band: int, keep_logical: bool = False + ) -> tuple[str, list[OTBObject], int]: + """This an internal function, only to be used by `build_fake_expressions`. Enable to create a fake expression just for one input and one band. + Regarding the "keep_logical" param: + - if True, for `input1 > input2`, returned fake expression is "str(input1) > str(input2)" + - if False, for `input1 > input2`, returned fake exp is "str(input1) > str(input2) ? 1 : 0"] Default False Args: x: input band: which band to consider (bands start at 1) keep_logical: whether to keep the logical expressions "as is" in case the input is a logical operation. - ex: if True, for `input1 > input2`, returned fake expression is "str(input1) > str(input2)" - if False, for `input1 > input2`, returned fake exp is "str(input1) > str(input2) ? 1 : 0". - Default False Returns: fake_exp: the fake expression for this band and input @@ -1116,182 +1391,385 @@ class Operation(App): """ # Special case for one-band slicer - if isinstance(x, Slicer) and hasattr(x, 'one_band_sliced'): - if keep_logical and isinstance(x.input, logicalOperation): + if isinstance(x, Slicer) and hasattr(x, "one_band_sliced"): + if keep_logical and isinstance(x.input, LogicalOperation): fake_exp = x.input.logical_fake_exp_bands[x.one_band_sliced - 1] - inputs = x.input.inputs - nb_channels = x.input.nb_channels + inputs, nb_channels = x.input.inputs, x.input.nb_channels elif isinstance(x.input, Operation): - # keep only one band of the expression + # Keep only one band of the expression fake_exp = x.input.fake_exp_bands[x.one_band_sliced - 1] - inputs = x.input.inputs - nb_channels = x.input.nb_channels + inputs, nb_channels = x.input.inputs, x.input.nb_channels else: # Add the band number (e.g. replace '<pyotb.App object>' by '<pyotb.App object>b1') - fake_exp = str(x.input) + f'b{x.one_band_sliced}' - inputs = [x.input] - nb_channels = {x.input: 1} - # For logicalOperation, we save almost the same attributes as an Operation - elif keep_logical and isinstance(x, logicalOperation): + fake_exp = f"{repr(x.input)}b{x.one_band_sliced}" + inputs, nb_channels = [x.input], {repr(x.input): 1} + # For LogicalOperation, we save almost the same attributes as an Operation + elif keep_logical and isinstance(x, LogicalOperation): fake_exp = x.logical_fake_exp_bands[band - 1] - inputs = x.inputs - nb_channels = x.nb_channels + inputs, nb_channels = x.inputs, x.nb_channels elif isinstance(x, Operation): fake_exp = x.fake_exp_bands[band - 1] - inputs = x.inputs - nb_channels = x.nb_channels + inputs, nb_channels = x.inputs, x.nb_channels # For int or float input, we just need to save their value elif isinstance(x, (int, float)): fake_exp = str(x) - inputs = None - nb_channels = None + inputs, nb_channels = None, None # We go on with other inputs, i.e. pyotb objects, filepaths... else: - nb_channels = {x: get_nbchannels(x)} - inputs = [x] # Add the band number (e.g. replace '<pyotb.App object>' by '<pyotb.App object>b1') - fake_exp = str(x) + f'b{band}' + fake_exp = f"{repr(x)}b{band}" + inputs, nb_channels = [x], {repr(x): get_nbchannels(x)} return fake_exp, inputs, nb_channels - def __str__(self): - """Return a nice string representation with object id.""" - return f'<pyotb.Operation `{self.operator}` object, id {id(self)}>' - + def __repr__(self) -> str: + """Return a nice string representation with operator and object id.""" + return f"<pyotb.Operation `{self.operator}` object, id {id(self)}>" -class logicalOperation(Operation): - """A specialization of Operation class for boolean logical operations i.e. >, <, >=, <=, ==, !=, `&` and `|`. - The only difference is that not only the BandMath expression is saved (e.g. "im1b1 > 0 ? 1 : 0"), but also the - logical expression (e.g. "im1b1 > 0") +class LogicalOperation(Operation): + """A specialization of Operation class for boolean logical operations. - """ + Supported operators are >, <, >=, <=, ==, !=, `&` and `|`. + The only difference is that not only the BandMath expression is saved + (e.g. "im1b1 > 0 ? 1 : 0"), but also the logical expression (e.g. "im1b1 > 0") - def __init__(self, operator, *inputs, nb_bands=None): - """Constructor for a logicalOperation object. - - Args: - operator: string operator (one of >, <, >=, <=, ==, !=, &, |) - *inputs: inputs - nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where - - """ - super().__init__(operator, *inputs, nb_bands=nb_bands) - self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands) + Args: + operator: string operator (one of >, <, >=, <=, ==, !=, &, |) + *inputs: inputs + nb_bands: optionally specify the output nb of bands - used only by pyotb.where - def create_fake_exp(self, operator, inputs, nb_bands=None): - """Create a 'fake' expression. + """ - E.g for the operation input1 > input2, we create a fake expression that is like - "str(input1) > str(input2) ? 1 : 0" and a logical fake expression that is like "str(input1) > str(input2)" + def __init__(self, operator: str, *inputs, nb_bands: int = None): + """Constructor for a LogicalOperation object.""" + self.logical_fake_exp_bands = [] + super().__init__(operator, *inputs, nb_bands=nb_bands, name="LogicalOperation") + self.logical_exp_bands, self.logical_exp = self.get_real_exp( + self.logical_fake_exp_bands + ) + + def build_fake_expressions( + self, + operator: str, + inputs: list[OTBObject | str | float], + nb_bands: int = None, + ): + """Create a list of 'fake' expressions, one for each band. + + For the operation input1 > input2, we create a fake expression like `str(input1) > str(input2) ? 1 : 0` + and a logical fake expression like `str(input1) > str(input2)` Args: operator: str (one of >, <, >=, <=, ==, !=, &, |) - inputs: Can be App, Output, Input, Operation, Slicer, filepath, int or float - nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where + inputs: Can be OTBObject, filepath, int or float + nb_bands: optionally specify the output nb of bands - used only internally by pyotb.where """ - # For any other operations, the output number of bands is the same as inputs - if any(isinstance(inp, Slicer) and hasattr(inp, 'one_band_sliced') for inp in inputs): - nb_bands = 1 - else: - nb_bands_list = [get_nbchannels(inp) for inp in inputs if not isinstance(inp, (float, int))] - # check that all inputs have the same nb of bands - if len(nb_bands_list) > 1: - if not all(x == nb_bands_list[0] for x in nb_bands_list): - raise Exception('All images do not have the same number of bands') - nb_bands = nb_bands_list[0] - # Create a list of fake exp, each item of the list corresponding to one band - for i, band in enumerate(range(1, nb_bands + 1)): - fake_exps = [] + for i, band in enumerate(range(1, self.get_nb_bands(inputs) + 1)): + expressions = [] for inp in inputs: - fake_exp, corresponding_inputs, nb_channels = super().create_one_input_fake_exp(inp, band, - keep_logical=True) - fake_exps.append(fake_exp) + fake_exp, corresp_inputs, nb_channels = super().make_fake_exp( + inp, band, keep_logical=True + ) + expressions.append(fake_exp) # Reference the inputs and nb of channels (only on first pass in the loop to avoid duplicates) - if i == 0 and corresponding_inputs and nb_channels: - self.inputs.extend(corresponding_inputs) + if i == 0 and corresp_inputs and nb_channels: + self.inputs.extend(corresp_inputs) self.nb_channels.update(nb_channels) - # We create here the "fake" expression. For example, for a BandMathX expression such as 'im1 > im2', # the logical fake expression stores the expression "str(input1) > str(input2)" - logical_fake_exp = f'({fake_exps[0]} {operator} {fake_exps[1]})' - + logical_fake_exp = f"({expressions[0]} {operator} {expressions[1]})" # We keep the logical expression, useful if later combined with other logical operations self.logical_fake_exp_bands.append(logical_fake_exp) # We create a valid BandMath expression, e.g. "str(input1) > str(input2) ? 1 : 0" - fake_exp = f'({logical_fake_exp} ? 1 : 0)' + fake_exp = f"({logical_fake_exp} ? 1 : 0)" self.fake_exp_bands.append(fake_exp) -def get_nbchannels(inp): +class Input(App): + """Class for transforming a filepath to pyotb object. + + Args: + filepath: Anything supported by GDAL (local file on the filesystem, remote resource, etc.) + + """ + + def __init__(self, filepath: str): + """Initialize an ExtractROI OTB app from a filepath, set dtype and store filepath.""" + super().__init__("ExtractROI", {"in": filepath}, quiet=True, frozen=True) + self._name = f"Input from {filepath}" + if not filepath.startswith(("/vsi", "http://", "https://", "ftp://")): + filepath = Path(filepath) + self.filepath = filepath + self.propagate_dtype() + self.execute() + + def __repr__(self) -> str: + """Return a string representation with file path, used in Operation to store file ref.""" + return f"<pyotb.Input object, from {self.filepath}>" + + +class Output(OTBObject): + """Object that behave like a pointer to a specific application in-memory output or file. + + Args: + pyotb_app: The pyotb App to store reference from + param_key: Output parameter key of the target app + filepath: path of the output file (if not memory) + mkdir: create missing parent directories + + """ + + _filepath: str | Path = None + + @deprecated_alias(app="pyotb_app", output_parameter_key="param_key") + def __init__( + self, + pyotb_app: App, + param_key: str = None, + filepath: str = None, + mkdir: bool = True, + ): + """Constructor for an Output object, initialized during App.__init__.""" + self.parent_pyotb_app = pyotb_app # keep trace of parent app + self.param_key = param_key + self.filepath = filepath + if mkdir and filepath is not None: + self.make_parent_dirs() + + @property + def name(self) -> str: + """Return Output name containing filepath.""" + return f"Output {self.param_key} from {self.parent_pyotb_app.name}" + + @property + def app(self) -> otb.Application: + """Reference to the parent pyotb otb.Application instance.""" + return self.parent_pyotb_app.app + + @property + @deprecated_attr(replacement="parent_pyotb_app") + def pyotb_app(self) -> App: + """Reference to the parent pyotb App (deprecated).""" + + @property + def exports_dic(self) -> dict[str, dict]: + """Reference to parent _exports_dic object that contains np array exports.""" + return self.parent_pyotb_app.exports_dic + + @property + def output_image_key(self) -> str: + """Force the right key to be used when accessing the OTBObject.""" + return self.param_key + + @property + def filepath(self) -> str | Path: + """Property to manage output URL.""" + if self._filepath is None: + raise ValueError("Filepath is not set") + return self._filepath + + @filepath.setter + def filepath(self, path: str): + if isinstance(path, str): + if path and not path.startswith(("/vsi", "http://", "https://", "ftp://")): + path = Path(path.split("?")[0]) + self._filepath = path + + def exists(self) -> bool: + """Check if the output file exist on disk. + + Raises: + ValueError: if filepath is not set or is remote URL + + """ + if not isinstance(self.filepath, Path): + raise ValueError("Filepath is not set or points to a remote URL") + return self.filepath.exists() + + def make_parent_dirs(self): + """Create missing parent directories. + + Raises: + ValueError: if filepath is not set or is remote URL + + """ + if not isinstance(self.filepath, Path): + raise ValueError("Filepath is not set or points to a remote URL") + self.filepath.parent.mkdir(parents=True, exist_ok=True) + + def write(self, filepath: None | str | Path = None, **kwargs) -> bool: + """Write output to disk, filepath is not required if it was provided to parent App during init. + + Args: + filepath: path of the output file, can be None if a value was passed during app init + + """ + if filepath is None: + return self.parent_pyotb_app.write( + {self.output_image_key: self.filepath}, **kwargs + ) + return self.parent_pyotb_app.write({self.output_image_key: filepath}, **kwargs) + + def __str__(self) -> str: + """Return string representation of Output filepath.""" + return str(self.filepath) + + +def get_nbchannels(inp: str | Path | OTBObject) -> int: """Get the nb of bands of input image. Args: - inp: can be filepath or pyotb object + inp: input file or OTBObject Returns: number of bands in image + Raises: + TypeError: if inp band count cannot be retrieved + """ - if isinstance(inp, otbObject): - nb_channels = inp.shape[-1] - else: + if isinstance(inp, OTBObject): + return inp.shape[-1] + if isinstance(inp, (str, Path)): # Executing the app, without printing its log try: info = App("ReadImageInfo", inp, quiet=True) - nb_channels = info.GetParameterInt("numberbands") - except Exception as e: # this happens when we pass a str that is not a filepath - raise TypeError(f'Could not get the number of channels of `{inp}`. Not a filepath or wrong filepath') from e - return nb_channels + return info["numberbands"] + except RuntimeError as info_err: # e.g. file is missing + raise TypeError( + f"Could not get the number of channels file '{inp}' ({info_err})" + ) from info_err + raise TypeError(f"Can't read number of channels of type '{type(inp)}' object {inp}") + +def get_pixel_type(inp: str | Path | OTBObject) -> str: + """Get the encoding of input image pixels as integer enum. -def get_pixel_type(inp): - """Get the encoding of input image pixels. + OTB enum e.g. `otbApplication.ImagePixelType_uint8'. + For an OTBObject with several outputs, only the pixel type of the first output is returned Args: - inp: can be filepath or pyotb object + inp: input file or OTBObject Returns: - pixel_type: OTB enum e.g. `otbApplication.ImagePixelType_uint8', which actually is an int. - For an App with several outputs, only the pixel type of the first output is returned + OTB enum + + Raises: + TypeError: if inp pixel type cannot be retrieved """ - if isinstance(inp, str): - # Executing the app, without printing its log + if isinstance(inp, OTBObject): + return inp.app.GetParameterOutputImagePixelType(inp.output_image_key) + if isinstance(inp, (str, Path)): try: info = App("ReadImageInfo", inp, quiet=True) - except Exception as info_err: # this happens when we pass a str that is not a filepath - raise TypeError(f'Could not get the pixel type of `{inp}`. Not a filepath or wrong filepath') from info_err - datatype = info.GetParameterString("datatype") # which is such as short, float... - if not datatype: - raise Exception(f'Unable to read pixel type of image {inp}') - datatype_to_pixeltype = {'unsigned_char': 'uint8', 'short': 'int16', 'unsigned_short': 'uint16', - 'int': 'int32', 'unsigned_int': 'uint32', 'long': 'int32', 'ulong': 'uint32', - 'float': 'float', 'double': 'double'} - pixel_type = datatype_to_pixeltype[datatype] - pixel_type = getattr(otb, f'ImagePixelType_{pixel_type}') - elif isinstance(inp, (otbObject)): - pixel_type = inp.GetParameterOutputImagePixelType(inp.output_param) - else: - raise TypeError(f'Could not get the pixel type. Not supported type: {inp}') - - return pixel_type - - -def parse_pixel_type(pixel_type): + datatype = info["datatype"] # which is such as short, float... + except ( + RuntimeError + ) as info_err: # this happens when we pass a str that is not a filepath + raise TypeError( + f"Could not get the pixel type of `{inp}` ({info_err})" + ) from info_err + if datatype: + return parse_pixel_type(datatype) + raise TypeError(f"Could not get the pixel type of {type(inp)} object {inp}") + + +def parse_pixel_type(pixel_type: str | int) -> int: """Convert one str pixel type to OTB integer enum if necessary. Args: - pixel_type: pixel type. can be str, int or dict + pixel_type: pixel type to parse Returns: - pixel_type integer value + pixel_type OTB enum integer value + + Raises: + KeyError: if pixel_type name is unknown + TypeError: if type(pixel_type) isn't int or str """ - if isinstance(pixel_type, str): # this correspond to 'uint8' etc... - return getattr(otb, f'ImagePixelType_{pixel_type}') - if isinstance(pixel_type, int): + if isinstance(pixel_type, int): # normal OTB int enum return pixel_type - raise ValueError(f'Bad pixel type specification ({pixel_type})') + if isinstance(pixel_type, str): # correspond to 'uint8' etc... + datatype_to_pixeltype = { + "unsigned_char": "uint8", + "short": "int16", + "unsigned_short": "uint16", + "int": "int32", + "unsigned_int": "uint32", + "long": "int32", + "ulong": "uint32", + "float": "float", + "double": "double", + } + if pixel_type in datatype_to_pixeltype.values(): + return getattr(otb, f"ImagePixelType_{pixel_type}") + if pixel_type in datatype_to_pixeltype: + return getattr(otb, f"ImagePixelType_{datatype_to_pixeltype[pixel_type]}") + raise KeyError( + f"Unknown dtype `{pixel_type}`. Available ones: {datatype_to_pixeltype}" + ) + raise TypeError( + f"Bad pixel type specification ({pixel_type} of type {type(pixel_type)})" + ) + + +def get_out_images_param_keys(otb_app: otb.Application) -> list[str]: + """Return every output parameter keys of a bare OTB app.""" + return [ + key + for key in otb_app.GetParametersKeys() + if otb_app.GetParameterType(key) == otb.ParameterType_OutputImage + ] + + +def summarize( + obj: App | Output | str | float | list, + strip_inpath: bool = False, + strip_outpath: bool = False, +) -> dict[str, dict | Any] | str | float | list: + """Recursively summarize parameters of an App or Output object and its parents. + + At the deepest recursion level, this function just return any parameter value, + path stripped if needed, or app summarized in case of a pipeline. + If strip_path is enabled, paths are truncated after the first "?" character. + Can be useful to remove URLs tokens from inputs (e.g. SAS or S3 credentials), + or extended filenames from outputs. + + Args: + obj: input object / parameter value to summarize + strip_inpath: strip all input paths + strip_outpath: strip all output paths + + Returns: + nested dictionary containing name and parameters of an app and its parents + + """ + if isinstance(obj, list): + return [summarize(o) for o in obj] + if isinstance(obj, Output): + return summarize(obj.parent_pyotb_app) + # => This is the deepest recursion level + if not isinstance(obj, App): + return obj + + def strip_path(param: str | Any): + if isinstance(param, list): + return [strip_path(p) for p in param] + if not isinstance(param, str): + return summarize(param) + return param.split("?")[0] + + # Call / top level of recursion : obj is an App + parameters = {} + # We need to return parameters values, summarized if param is an App + for key, param in obj.parameters.items(): + if strip_inpath and obj.is_input(key) or strip_outpath and obj.is_output(key): + parameters[key] = strip_path(param) + else: + parameters[key] = summarize(param) + return {"name": obj.app.GetName(), "parameters": parameters} diff --git a/pyotb/depreciation.py b/pyotb/depreciation.py new file mode 100644 index 0000000000000000000000000000000000000000..6794373dbf621315a9d3eaaa6263efe532d2ca23 --- /dev/null +++ b/pyotb/depreciation.py @@ -0,0 +1,114 @@ +"""Helps with deprecated classes and methods. + +Taken from https://stackoverflow.com/questions/49802412/how-to-implement-deprecation-in-python-with-argument-alias +""" +from typing import Callable, Dict, Any +import functools +import warnings + + +WARN = "\033[91m" +ENDC = "\033[0m" +OKAY = "\033[92m" + + +def depreciation_warning(message: str): + """Shows a warning message. + + Args: + message: message to log + + """ + warnings.warn( + message=message, + category=DeprecationWarning, + stacklevel=3, + ) + + +def deprecated_alias(**aliases: str) -> Callable: + """Decorator for deprecated function and method arguments. + + Use as follows: + + @deprecated_alias(old_arg='new_arg') + def myfunc(new_arg): + ... + + Args: + **aliases: aliases + + Returns: + wrapped function + + """ + + def deco(func: Callable): + @functools.wraps(func) + def wrapper(*args, **kwargs): + rename_kwargs(func.__name__, kwargs, aliases) + return func(*args, **kwargs) + + return wrapper + + return deco + + +def rename_kwargs(func_name: str, kwargs: Dict[str, Any], aliases: Dict[str, str]): + """Helper function for deprecating function arguments. + + Args: + func_name: function + kwargs: keyword args + aliases: aliases + + Raises: + ValueError: if both old and new arguments are provided + + """ + for alias, new in aliases.items(): + if alias in kwargs: + if new in kwargs: + raise ValueError( + f"{func_name} received both {alias} and {new} as arguments!" + f" {alias} is deprecated, use {new} instead." + ) + message = ( + f"{WARN}`{alias}`{ENDC} is deprecated as an argument to " + f"`{func_name}`; use {OKAY}`{new}`{ENDC} instead." + ) + depreciation_warning(message) + kwargs[new] = kwargs.pop(alias) + + +def deprecated_attr(replacement: str) -> Callable: + """Decorator for deprecated attr. + + Use as follows: + + @deprecated_attr(replacement='new_attr') + def old_attr(...): + ... + + Args: + replacement: name of the new attr (method or attribute) + + Returns: + wrapped function + + """ + + def deco(attr: Any): + @functools.wraps(attr) + def wrapper(self, *args, **kwargs): + depreciation_warning( + f"{WARN}`{attr.__name__}`{ENDC} will be removed in future " + f"releases. Please replace {WARN}`{attr.__name__}`{ENDC} with " + f"{OKAY}`{replacement}`{ENDC}." + ) + out = getattr(self, replacement) + return out(*args, **kwargs) if isinstance(out, Callable) else out + + return wrapper + + return deco diff --git a/pyotb/functions.py b/pyotb/functions.py index eee401a3ba6638df6e86f9efe96cc8238c9a4655..fe93641f663dff43f45b749a922a8feb8eaa31d7 100644 --- a/pyotb/functions.py +++ b/pyotb/functions.py @@ -1,28 +1,35 @@ -# -*- coding: utf-8 -*- """This module provides a set of functions for pyotb.""" +from __future__ import annotations + import inspect import os +import subprocess import sys import textwrap import uuid from collections import Counter +from pathlib import Path -from .core import (otbObject, App, Input, Operation, logicalOperation, get_nbchannels) +from .core import App, Input, LogicalOperation, Operation, get_nbchannels from .helpers import logger -def where(cond, x, y): +def where(cond: App | str, x: App | str | float, y: App | str | float) -> Operation: """Functionally similar to numpy.where. Where cond is True (!=0), returns x. Else returns y. + If cond is monoband whereas x or y are multiband, cond channels are expanded to match x & y ones. + Args: - cond: condition, must be a raster (filepath, App, Operation...). If cond is monoband whereas x or y are - multiband, cond channels are expanded to match x & y ones. - x: value if cond is True. Can be float, int, App, filepath, Operation... - y: value if cond is False. Can be float, int, App, filepath, Operation... + cond: condition, must be a raster (filepath, App, Operation...). + x: value if cond is True. Can be: float, int, App, filepath, Operation... + y: value if cond is False. Can be: float, int, App, filepath, Operation... Returns: an output where pixels are x if cond is True, else y + Raises: + ValueError: if x and y have different number of bands + """ # Checking the number of bands of rasters. Several cases : # - if cond is monoband, x and y can be multibands. Then cond will adapt to match x and y nb of bands @@ -32,62 +39,61 @@ def where(cond, x, y): x_nb_channels = get_nbchannels(x) if not isinstance(y, (int, float)): y_nb_channels = get_nbchannels(y) - if x_nb_channels and y_nb_channels: if x_nb_channels != y_nb_channels: - raise ValueError('X and Y images do not have the same number of bands. ' - f'X has {x_nb_channels} bands whereas Y has {y_nb_channels} bands') + raise ValueError( + "X and Y images do not have the same number of bands. " + f"X has {x_nb_channels} bands whereas Y has {y_nb_channels} bands" + ) x_or_y_nb_channels = x_nb_channels if x_nb_channels else y_nb_channels cond_nb_channels = get_nbchannels(cond) - - # Get the number of bands of the result - if x_or_y_nb_channels: # if X or Y is a raster - out_nb_channels = x_or_y_nb_channels - else: # if only cond is a raster - out_nb_channels = cond_nb_channels - - if cond_nb_channels != 1 and x_or_y_nb_channels and cond_nb_channels != x_or_y_nb_channels: - raise ValueError('Condition and X&Y do not have the same number of bands. Condition has ' - f'{cond_nb_channels} bands whereas X&Y have {x_or_y_nb_channels} bands') - + if ( + cond_nb_channels != 1 + and x_or_y_nb_channels + and cond_nb_channels != x_or_y_nb_channels + ): + raise ValueError( + "Condition and X&Y do not have the same number of bands. Condition has " + f"{cond_nb_channels} bands whereas X&Y have {x_or_y_nb_channels} bands" + ) # If needed, duplicate the single band binary mask to multiband to match the dimensions of x & y if cond_nb_channels == 1 and x_or_y_nb_channels and x_or_y_nb_channels != 1: - logger.info('The condition has one channel whereas X/Y has/have %s channels. Expanding number' - ' of channels of condition to match the number of channels of X/Y', x_or_y_nb_channels) - - operation = Operation('?', cond, x, y, nb_bands=out_nb_channels) + logger.info( + "The condition has one channel whereas X/Y has/have %s channels. Expanding number" + " of channels of condition to match the number of channels of X/Y", + x_or_y_nb_channels, + ) + # Get the number of bands of the result + out_nb_channels = x_or_y_nb_channels or cond_nb_channels - return operation + return Operation("?", cond, x, y, nb_bands=out_nb_channels) -def clip(a, a_min, a_max): +def clip(image: App | str, v_min: App | str | float, v_max: App | str | float): """Clip values of image in a range of values. Args: - a: input raster, can be filepath or any pyotb object - a_min: minimum value of the range - a_max: maximum value of the range + image: input raster, can be filepath or any pyotb object + v_min: minimum value of the range + v_max: maximum value of the range Returns: raster whose values are clipped in the range """ - if isinstance(a, str): - a = Input(a) - - res = where(a <= a_min, a_min, - where(a >= a_max, a_max, a)) - return res + if isinstance(image, (str, Path)): + image = Input(image) + return where(image <= v_min, v_min, where(image >= v_max, v_max, image)) def all(*inputs): # pylint: disable=redefined-builtin """Check if value is different than 0 everywhere along the band axis. - For only one image, this function checks that all bands of the image are True (i.e. !=0) and outputs - a singleband boolean raster + For only one image, this function checks that all bands of the image are True (i.e. !=0) + and outputs a singleband boolean raster For several images, this function checks that all images are True (i.e. !=0) and outputs - a boolean raster, with as many bands as the inputs + a boolean raster, with as many bands as the inputs Args: inputs: inputs can be 1) a single image or 2) several images, either passed as separate arguments @@ -100,40 +106,40 @@ def all(*inputs): # pylint: disable=redefined-builtin # If necessary, flatten inputs if len(inputs) == 1 and isinstance(inputs[0], (list, tuple)): inputs = inputs[0] - # Add support for generator inputs (to have the same behavior as built-in `all` function) - if isinstance(inputs, tuple) and len(inputs) == 1 and inspect.isgenerator(inputs[0]): + if ( + isinstance(inputs, tuple) + and len(inputs) == 1 + and inspect.isgenerator(inputs[0]) + ): inputs = list(inputs[0]) - # Transforming potential filepaths to pyotb objects inputs = [Input(inp) if isinstance(inp, str) else inp for inp in inputs] # Checking that all bands of the single image are True if len(inputs) == 1: inp = inputs[0] - if isinstance(inp, logicalOperation): + if isinstance(inp, LogicalOperation): res = inp[:, :, 0] else: - res = (inp[:, :, 0] != 0) - + res = inp[:, :, 0] != 0 for band in range(1, inp.shape[-1]): - if isinstance(inp, logicalOperation): + if isinstance(inp, LogicalOperation): res = res & inp[:, :, band] else: res = res & (inp[:, :, band] != 0) + return res # Checking that all images are True + if isinstance(inputs[0], LogicalOperation): + res = inputs[0] else: - if isinstance(inputs[0], logicalOperation): - res = inputs[0] + res = inputs[0] != 0 + for inp in inputs[1:]: + if isinstance(inp, LogicalOperation): + res = res & inp else: - res = (inputs[0] != 0) - for inp in inputs[1:]: - if isinstance(inp, logicalOperation): - res = res & inp - else: - res = res & (inp != 0) - + res = res & (inp != 0) return res @@ -148,6 +154,7 @@ def any(*inputs): # pylint: disable=redefined-builtin Args: inputs: inputs can be 1) a single image or 2) several images, either passed as separate arguments or inside a list + Returns: OR intersection @@ -155,40 +162,41 @@ def any(*inputs): # pylint: disable=redefined-builtin # If necessary, flatten inputs if len(inputs) == 1 and isinstance(inputs[0], (list, tuple)): inputs = inputs[0] - # Add support for generator inputs (to have the same behavior as built-in `any` function) - if isinstance(inputs, tuple) and len(inputs) == 1 and inspect.isgenerator(inputs[0]): + if ( + isinstance(inputs, tuple) + and len(inputs) == 1 + and inspect.isgenerator(inputs[0]) + ): inputs = list(inputs[0]) - # Transforming potential filepaths to pyotb objects inputs = [Input(inp) if isinstance(inp, str) else inp for inp in inputs] # Checking that at least one band of the image is True if len(inputs) == 1: inp = inputs[0] - if isinstance(inp, logicalOperation): + if isinstance(inp, LogicalOperation): res = inp[:, :, 0] else: - res = (inp[:, :, 0] != 0) + res = inp[:, :, 0] != 0 for band in range(1, inp.shape[-1]): - if isinstance(inp, logicalOperation): + if isinstance(inp, LogicalOperation): res = res | inp[:, :, band] else: res = res | (inp[:, :, band] != 0) + return res # Checking that at least one image is True + if isinstance(inputs[0], LogicalOperation): + res = inputs[0] else: - if isinstance(inputs[0], logicalOperation): - res = inputs[0] + res = inputs[0] != 0 + for inp in inputs[1:]: + if isinstance(inp, LogicalOperation): + res = res | inp else: - res = (inputs[0] != 0) - for inp in inputs[1:]: - if isinstance(inp, logicalOperation): - res = res | inp - else: - res = res | (inp != 0) - + res = res | (inp != 0) return res @@ -212,13 +220,19 @@ def run_tf_function(func): Returns: wrapper: a function that returns a pyotb object + Raises: + SystemError: if OTBTF apps are missing + """ try: - from .apps import TensorflowModelServe - except ImportError: - logger.error('Could not run Tensorflow function: failed to import TensorflowModelServe.' - 'Check that you have OTBTF configured (https://github.com/remicres/otbtf#how-to-install)') - raise + from .apps import ( # pylint: disable=import-outside-toplevel + TensorflowModelServe, + ) + except ImportError as err: + raise SystemError( + "Could not run Tensorflow function: failed to import TensorflowModelServe." + "Check that you have OTBTF configured (https://github.com/remicres/otbtf#how-to-install)" + ) from err def get_tf_pycmd(output_dir, channels, scalar_inputs): """Create a string containing all python instructions necessary to create and save the Keras model. @@ -233,14 +247,14 @@ def run_tf_function(func): """ # Getting the string definition of the tf function (e.g. "def multiply(x1, x2):...") - # TODO: maybe not entirely foolproof, maybe we should use dill instead? but it would add a dependency + # Maybe not entirely foolproof, maybe we should use dill instead? but it would add a dependency func_def_str = inspect.getsource(func) func_name = func.__name__ create_and_save_model_str = func_def_str - # Adding the instructions to create the model and save it to output dir - create_and_save_model_str += textwrap.dedent(f""" + create_and_save_model_str += textwrap.dedent( + f""" import tensorflow as tf model_inputs = [] @@ -260,11 +274,12 @@ def run_tf_function(func): # Create and save the .pb model model = tf.keras.Model(inputs=model_inputs, outputs=output) model.save("{output_dir}") - """) + """ + ) return create_and_save_model_str - def wrapper(*inputs, tmp_dir='/tmp'): + def wrapper(*inputs, tmp_dir="/tmp"): """For the user point of view, this function simply applies some TensorFlow operations to some rasters. Implicitly, it saves a .pb model that describe the TF operations, then creates an OTB ModelServe application @@ -284,45 +299,64 @@ def run_tf_function(func): raster_inputs = [] for inp in inputs: try: - # this is for raster input + # This is for raster input channel = get_nbchannels(inp) channels.append(channel) scalar_inputs.append(None) raster_inputs.append(inp) except TypeError: - # this is for other inputs (float, int) + # This is for other inputs (float, int) channels.append(None) scalar_inputs.append(inp) # Create and save the model. This is executed **inside an independent process** because (as of 2022-03), # tensorflow python library and OTBTF are incompatible - out_savedmodel = os.path.join(tmp_dir, f'tmp_otbtf_model_{uuid.uuid4()}') + out_savedmodel = os.path.join(tmp_dir, f"tmp_otbtf_model_{uuid.uuid4()}") pycmd = get_tf_pycmd(out_savedmodel, channels, scalar_inputs) cmd_args = [sys.executable, "-c", pycmd] + # TODO: remove subprocess execution since this issues has been fixed with OTBTF 4.0 try: - import subprocess - subprocess.run(cmd_args, env=os.environ, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) + subprocess.run( + cmd_args, + env=os.environ, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + ) except subprocess.SubprocessError: logger.debug("Failed to call subprocess") if not os.path.isdir(out_savedmodel): logger.info("Failed to save the model") # Initialize the OTBTF model serving application - model_serve = TensorflowModelServe({'model.dir': out_savedmodel, 'optim.disabletiling': 'on', - 'model.fullyconv': 'on'}, n_sources=len(raster_inputs), frozen=True) + model_serve = TensorflowModelServe( + { + "model.dir": out_savedmodel, + "optim.disabletiling": "on", + "model.fullyconv": "on", + }, + n_sources=len(raster_inputs), + frozen=True, + ) # Set parameters and execute for i, inp in enumerate(raster_inputs): - model_serve.set_parameters({f'source{i + 1}.il': [inp]}) + model_serve.set_parameters({f"source{i + 1}.il": [inp]}) model_serve.execute() - # TODO: handle the deletion of the temporary model ? + # Possible ENH: handle the deletion of the temporary model ? return model_serve return wrapper -def define_processing_area(*args, window_rule='intersection', pixel_size_rule='minimal', interpolator='nn', - reference_window_input=None, reference_pixel_size_input=None): +def define_processing_area( + *args, + window_rule: str = "intersection", + pixel_size_rule: str = "minimal", + interpolator: str = "nn", + reference_window_input: dict = None, + reference_pixel_size_input: str = None, +) -> list[App]: """Given several inputs, this function handles the potential resampling and cropping to same extent. WARNING: Not fully implemented / tested @@ -346,142 +380,168 @@ def define_processing_area(*args, window_rule='intersection', pixel_size_rule='m inputs.extend(arg) else: inputs.append(arg) - # Getting metadatas of inputs metadatas = {} for inp in inputs: if isinstance(inp, str): # this is for filepaths - metadata = Input(inp).GetImageMetaData('out') - elif isinstance(inp, otbObject): - metadata = inp.GetImageMetaData(inp.output_param) + metadata = Input(inp).app.GetImageMetaData("out") + elif isinstance(inp, App): + metadata = inp.app.GetImageMetaData(inp.output_param) else: raise TypeError(f"Wrong input : {inp}") metadatas[inp] = metadata # Get a metadata of an arbitrary image. This is just to compare later with other images any_metadata = next(iter(metadatas.values())) - # Checking if all images have the same projection - if not all(metadata['ProjectionRef'] == any_metadata['ProjectionRef'] - for metadata in metadatas.values()): - logger.warning('All images may not have the same CRS, which might cause unpredictable results') + if not all( + metadata["ProjectionRef"] == any_metadata["ProjectionRef"] + for metadata in metadatas.values() + ): + logger.warning( + "All images may not have the same CRS, which might cause unpredictable results" + ) # Handling different spatial footprints - # TODO: there seems to have a bug, ImageMetaData is not updated when running an app, + # TODO: find possible bug - ImageMetaData is not updated when running an app # cf https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/2234. Should we use ImageOrigin instead? - if not all(metadata['UpperLeftCorner'] == any_metadata['UpperLeftCorner'] - and metadata['LowerRightCorner'] == any_metadata['LowerRightCorner'] - for metadata in metadatas.values()): + if not all( + md["UpperLeftCorner"] == any_metadata["UpperLeftCorner"] + and md["LowerRightCorner"] == any_metadata["LowerRightCorner"] + for md in metadatas.values() + ): # Retrieving the bounding box that will be common for all inputs - if window_rule == 'intersection': + if window_rule == "intersection": # The coordinates depend on the orientation of the axis of projection - if any_metadata['GeoTransform'][1] >= 0: - ulx = max(metadata['UpperLeftCorner'][0] for metadata in metadatas.values()) - lrx = min(metadata['LowerRightCorner'][0] for metadata in metadatas.values()) + if any_metadata["GeoTransform"][1] >= 0: + ulx = max(md["UpperLeftCorner"][0] for md in metadatas.values()) + lrx = min(md["LowerRightCorner"][0] for md in metadatas.values()) else: - ulx = min(metadata['UpperLeftCorner'][0] for metadata in metadatas.values()) - lrx = max(metadata['LowerRightCorner'][0] for metadata in metadatas.values()) - if any_metadata['GeoTransform'][-1] >= 0: - lry = min(metadata['LowerRightCorner'][1] for metadata in metadatas.values()) - uly = max(metadata['UpperLeftCorner'][1] for metadata in metadatas.values()) + ulx = min(md["UpperLeftCorner"][0] for md in metadatas.values()) + lrx = max(md["LowerRightCorner"][0] for md in metadatas.values()) + if any_metadata["GeoTransform"][-1] >= 0: + lry = min(md["LowerRightCorner"][1] for md in metadatas.values()) + uly = max(md["UpperLeftCorner"][1] for md in metadatas.values()) else: - lry = max(metadata['LowerRightCorner'][1] for metadata in metadatas.values()) - uly = min(metadata['UpperLeftCorner'][1] for metadata in metadatas.values()) - - elif window_rule == 'same_as_input': - ulx = metadatas[reference_window_input]['UpperLeftCorner'][0] - lrx = metadatas[reference_window_input]['LowerRightCorner'][0] - lry = metadatas[reference_window_input]['LowerRightCorner'][1] - uly = metadatas[reference_window_input]['UpperLeftCorner'][1] - elif window_rule == 'specify': - pass - # TODO : it is when the user explicitly specifies the bounding box -> add some arguments in the function - elif window_rule == 'union': - pass - # TODO : it is when the user wants the final bounding box to be the union of all bounding box + lry = max(md["LowerRightCorner"][1] for md in metadatas.values()) + uly = min(md["UpperLeftCorner"][1] for md in metadatas.values()) + + elif window_rule == "same_as_input": + ulx = metadatas[reference_window_input]["UpperLeftCorner"][0] + lrx = metadatas[reference_window_input]["LowerRightCorner"][0] + lry = metadatas[reference_window_input]["LowerRightCorner"][1] + uly = metadatas[reference_window_input]["UpperLeftCorner"][1] + elif window_rule == "specify": + # When the user explicitly specifies the bounding box -> add some arguments in the function + ... + elif window_rule == "union": + # When the user wants the final bounding box to be the union of all bounding box # It should replace any 'outside' pixel by some NoData -> add `fillvalue` argument in the function - - logger.info('Cropping all images to extent Upper Left (%s, %s), Lower Right (%s, %s)', ulx, uly, lrx, lry) + ... # Applying this bounding box to all inputs + bounds = (ulx, uly, lrx, lry) + logger.info( + "Cropping all images to extent Upper Left (%s, %s), Lower Right (%s, %s)", + *bounds, + ) new_inputs = [] for inp in inputs: try: params = { - 'in': inp, 'mode': 'extent', 'mode.extent.unit': 'phy', - 'mode.extent.ulx': ulx, 'mode.extent.uly': lry, # bug in OTB <= 7.3 : - 'mode.extent.lrx': lrx, 'mode.extent.lry': uly, # ULY/LRY are inverted + "in": inp, + "mode": "extent", + "mode.extent.unit": "phy", + "mode.extent.ulx": ulx, + "mode.extent.uly": uly, + "mode.extent.lrx": lrx, + "mode.extent.lry": lry, } - new_input = App('ExtractROI', params) - # TODO: OTB 7.4 fixes this bug, how to handle different versions of OTB? + new_input = App("ExtractROI", params, quiet=True) new_inputs.append(new_input) # Potentially update the reference inputs for later resampling - if str(inp) == str(reference_pixel_size_input): # we use comparison of string because calling '==' + if str(inp) == str(reference_pixel_size_input): + # We use comparison of string because calling '==' # on pyotb objects implicitly calls BandMathX application, which is not desirable reference_pixel_size_input = new_input - except RuntimeError as e: - logger.error('Cannot define the processing area for input %s: %s', inp, e) - raise + except RuntimeError as err: + raise ValueError( + f"Cannot define the processing area for input {inp}" + ) from err inputs = new_inputs - # Update metadatas - metadatas = {input: input.GetImageMetaData('out') for input in inputs} + metadatas = {input: input.app.GetImageMetaData("out") for input in inputs} # Get a metadata of an arbitrary image. This is just to compare later with other images any_metadata = next(iter(metadatas.values())) - # Handling different pixel sizes - if not all(metadata['GeoTransform'][1] == any_metadata['GeoTransform'][1] - and metadata['GeoTransform'][5] == any_metadata['GeoTransform'][5] - for metadata in metadatas.values()): + if not all( + md["GeoTransform"][1] == any_metadata["GeoTransform"][1] + and md["GeoTransform"][5] == any_metadata["GeoTransform"][5] + for md in metadatas.values() + ): # Retrieving the pixel size that will be common for all inputs - if pixel_size_rule == 'minimal': + if pixel_size_rule == "minimal": # selecting the input with the smallest x pixel size - reference_input = min(metadatas, key=lambda x: metadatas[x]['GeoTransform'][1]) - if pixel_size_rule == 'maximal': + reference_input = min( + metadatas, key=lambda x: metadatas[x]["GeoTransform"][1] + ) + if pixel_size_rule == "maximal": # selecting the input with the highest x pixel size - reference_input = max(metadatas, key=lambda x: metadatas[x]['GeoTransform'][1]) - elif pixel_size_rule == 'same_as_input': + reference_input = max( + metadatas, key=lambda x: metadatas[x]["GeoTransform"][1] + ) + elif pixel_size_rule == "same_as_input": reference_input = reference_pixel_size_input - elif pixel_size_rule == 'specify': - pass - # TODO : when the user explicitly specify the pixel size -> add argument inside the function - pixel_size = metadatas[reference_input]['GeoTransform'][1] - logger.info('Resampling all inputs to resolution: %s', pixel_size) + elif pixel_size_rule == "specify": + # When the user explicitly specify the pixel size -> add argument inside the function + ... + + pixel_size = metadatas[reference_input]["GeoTransform"][1] # Perform resampling on inputs that do not comply with the target pixel size + logger.info("Resampling all inputs to resolution: %s", pixel_size) new_inputs = [] for inp in inputs: - if metadatas[inp]['GeoTransform'][1] != pixel_size: - superimposed = App('Superimpose', inr=reference_input, inm=inp, interpolator=interpolator) + if metadatas[inp]["GeoTransform"][1] != pixel_size: + superimposed = App( + "Superimpose", + inr=reference_input, + inm=inp, + interpolator=interpolator, + ) new_inputs.append(superimposed) else: new_inputs.append(inp) inputs = new_inputs - - # Update metadatas - metadatas = {inp: inp.GetImageMetaData('out') for inp in inputs} + metadatas = {inp: inp.app.GetImageMetaData("out") for inp in inputs} # Final superimposition to be sure to have the exact same image sizes - # Getting the sizes of images image_sizes = {} for inp in inputs: if isinstance(inp, str): inp = Input(inp) image_sizes[inp] = inp.shape[:2] - # Selecting the most frequent image size. It will be used as reference. most_common_image_size, _ = Counter(image_sizes.values()).most_common(1)[0] - same_size_images = [inp for inp, image_size in image_sizes.items() if image_size == most_common_image_size] + same_size_images = [ + inp + for inp, image_size in image_sizes.items() + if image_size == most_common_image_size + ] # Superimposition for images that do not have the same size as the others new_inputs = [] for inp in inputs: if image_sizes[inp] != most_common_image_size: - superimposed = App('Superimpose', inr=same_size_images[0], inm=inp, interpolator=interpolator) + superimposed = App( + "Superimpose", + inr=same_size_images[0], + inm=inp, + interpolator=interpolator, + ) new_inputs.append(superimposed) else: new_inputs.append(inp) - inputs = new_inputs - return inputs + return new_inputs diff --git a/pyotb/helpers.py b/pyotb/helpers.py index 742fee18be4bfbf6951d8e754cb102d470b16b4a..0e6ea2a06eee8b1b7e8cb7502d03b95861c4ef7f 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -1,24 +1,30 @@ -# -*- coding: utf-8 -*- -"""This module provides some helpers to properly initialize pyotb.""" +"""This module ensure we properly initialize pyotb, or raise SystemExit in case of broken install.""" +import logging import os import sys -import logging +import sysconfig from pathlib import Path from shutil import which +from .install import install_otb, interactive_config # Allow user to switch between OTB directories without setting every env variable OTB_ROOT = os.environ.get("OTB_ROOT") +DOCS_URL = "https://www.orfeo-toolbox.org/CookBook/Installation.html" # Logging # User can also get logger with `logging.getLogger("pyOTB")` # then use pyotb.set_logger_level() to adjust logger verbosity -logger = logging.getLogger("pyOTB") +logger = logging.getLogger("pyotb") logger_handler = logging.StreamHandler(sys.stdout) -formatter = logging.Formatter(fmt="%(asctime)s (%(levelname)-4s) [pyOTB] %(message)s", datefmt="%Y-%m-%d %H:%M:%S") +formatter = logging.Formatter( + fmt="%(asctime)s (%(levelname)-4s) [pyotb] %(message)s", datefmt="%Y-%m-%d %H:%M:%S" +) logger_handler.setFormatter(formatter) -# Search for PYOTB_LOGGER_LEVEL, else use OTB_LOGGER_LEVEL as pyOTB level, or fallback to INFO -LOG_LEVEL = os.environ.get("PYOTB_LOGGER_LEVEL") or os.environ.get("OTB_LOGGER_LEVEL") or "INFO" +# Search for PYOTB_LOGGER_LEVEL, else use OTB_LOGGER_LEVEL as pyotb level, or fallback to INFO +LOG_LEVEL = ( + os.environ.get("PYOTB_LOGGER_LEVEL") or os.environ.get("OTB_LOGGER_LEVEL") or "INFO" +) logger.setLevel(getattr(logging, LOG_LEVEL)) # Here it would be possible to use a different level for a specific handler # A more verbose one can go to text file while print only errors to stdout @@ -26,7 +32,7 @@ logger_handler.setLevel(getattr(logging, LOG_LEVEL)) logger.addHandler(logger_handler) -def set_logger_level(level): +def set_logger_level(level: str): """Allow user to change the current logging level. Args: @@ -36,22 +42,28 @@ def set_logger_level(level): logger_handler.setLevel(getattr(logging, level)) -def find_otb(prefix=OTB_ROOT, scan=True, scan_userdir=True): - """Try to load OTB bindings or scan system, help user in case of failure, set env variables. +def find_otb(prefix: str = OTB_ROOT, scan: bool = True): + """Try to load OTB bindings or scan system, help user in case of failure, set env. - Path precedence : OTB_ROOT > python bindings directory - OR search for releases installations : HOME - OR (for linux) : /opt/otbtf > /opt/otb > /usr/local > /usr - OR (for windows) : C:/Program Files + If in interactive prompt, user will be asked if he wants to install OTB. + The OTB_ROOT variable allow one to override default OTB version, with auto env setting. + Path precedence : $OTB_ROOT > location of python bindings location + Then, if OTB is not found: + search for releases installations: $HOME/Applications + OR (for Linux): /opt/otbtf > /opt/otb > /usr/local > /usr + OR (for Windows): C:/Program Files Args: prefix: prefix to search OTB in (Default value = OTB_ROOT) scan: find otb in system known locations (Default value = True) - scan_userdir: search for OTB release in user's home directory (Default value = True) Returns: otbApplication module + Raises: + SystemError: is OTB is not found (when using interactive mode) + SystemExit: if OTB is not found, since pyotb won't be usable + """ otb = None # Try OTB_ROOT env variable first (allow override default OTB version) @@ -59,51 +71,67 @@ def find_otb(prefix=OTB_ROOT, scan=True, scan_userdir=True): try: set_environment(prefix) import otbApplication as otb # pylint: disable=import-outside-toplevel + return otb - except EnvironmentError as e: + except SystemError as e: raise SystemExit(f"Failed to import OTB with prefix={prefix}") from e except ImportError as e: __suggest_fix_import(str(e), prefix) raise SystemExit("Failed to import OTB. Exiting.") from e # Else try import from actual Python path try: - # Here, we can't properly set env variables before OTB import. We assume user did this before running python + # Here, we can't properly set env variables before OTB import. + # We assume user did this before running python # For LD_LIBRARY_PATH problems, use OTB_ROOT instead of PYTHONPATH import otbApplication as otb # pylint: disable=import-outside-toplevel + if "OTB_APPLICATION_PATH" not in os.environ: lib_dir = __find_lib(otb_module=otb) apps_path = __find_apps_path(lib_dir) otb.Registry.SetApplicationPath(apps_path) return otb except ImportError as e: - PYTHONPATH = os.environ.get("PYTHONPATH") + pythonpath = os.environ.get("PYTHONPATH") if not scan: - raise SystemExit(f"Failed to import OTB with env PYTHONPATH={PYTHONPATH}") from e + raise SystemExit( + f"Failed to import OTB with env PYTHONPATH={pythonpath}" + ) from e # Else search system logger.info("Failed to import OTB. Searching for it...") - prefix = __find_otb_root(scan_userdir) - # Try to import one last time before raising error + prefix = __find_otb_root() + # Try auto install if shell is interactive + if not prefix and hasattr(sys, "ps1"): + if input("OTB is missing. Do you want to install it ? (y/n): ") == "y": + return find_otb(install_otb(*interactive_config())) + raise SystemError("OTB libraries not found on disk. ") + if not prefix: + raise SystemExit( + "OTB libraries not found on disk. " + "To install it, open an interactive python shell and 'import pyotb'" + ) + # If OTB was found on disk, set env and try to import one last time try: set_environment(prefix) import otbApplication as otb # pylint: disable=import-outside-toplevel + return otb - except EnvironmentError as e: + except SystemError as e: raise SystemExit("Auto setup for OTB env failed. Exiting.") from e - # Unknown error - except ModuleNotFoundError as e: - raise SystemExit("Can't run without OTB installed. Exiting.") from e # Help user to fix this except ImportError as e: __suggest_fix_import(str(e), prefix) raise SystemExit("Failed to import OTB. Exiting.") from e -def set_environment(prefix): +def set_environment(prefix: str): """Set environment variables (before OTB import), raise error if anything is wrong. Args: prefix: path to OTB root directory + Raises: + SystemError: if OTB or GDAL is not found + """ logger.info("Preparing environment for OTB in %s", prefix) # OTB root directory @@ -111,33 +139,36 @@ def set_environment(prefix): if not prefix.exists(): raise FileNotFoundError(str(prefix)) built_from_source = False - if not (prefix / 'README').exists(): + if not (prefix / "README").exists(): built_from_source = True # External libraries lib_dir = __find_lib(prefix) if not lib_dir: - raise EnvironmentError("Can't find OTB external libraries") - # This does not seems to work + raise SystemError("Can't find OTB external libraries") + # LD library path : this does not seems to work if sys.platform == "linux" and built_from_source: new_ld_path = f"{lib_dir}:{os.environ.get('LD_LIBRARY_PATH') or ''}" os.environ["LD_LIBRARY_PATH"] = new_ld_path + # Add python bindings directory first in PYTHONPATH otb_api = __find_python_api(lib_dir) if not otb_api: - raise EnvironmentError("Can't find OTB Python API") + raise SystemError("Can't find OTB Python API") if otb_api not in sys.path: sys.path.insert(0, otb_api) - # Add /bin first in PATH, in order to avoid conflicts with another GDAL install when using os.system() + + # Add /bin first in PATH, in order to avoid conflicts with another GDAL install os.environ["PATH"] = f"{prefix / 'bin'}{os.pathsep}{os.environ['PATH']}" - # Applications path (this can be tricky since OTB import will succeed even without apps) + # Ensure APPLICATION_PATH is set apps_path = __find_apps_path(lib_dir) if Path(apps_path).exists(): os.environ["OTB_APPLICATION_PATH"] = apps_path else: - raise EnvironmentError("Can't find OTB applications directory") - + raise SystemError("Can't find OTB applications directory") os.environ["LC_NUMERIC"] = "C" os.environ["GDAL_DRIVER_PATH"] = "disable" + + # Find GDAL libs if (prefix / "share/gdal").exists(): # Local GDAL (OTB Superbuild, .run, .exe) gdal_data = str(prefix / "share/gdal") @@ -150,25 +181,26 @@ def set_environment(prefix): gdal_data = str(prefix / "share/data") proj_lib = str(prefix / "share/proj") else: - raise EnvironmentError(f"Can't find GDAL location with current OTB prefix '{prefix}' or in /usr") - + raise SystemError( + f"Can't find GDAL location with current OTB prefix '{prefix}' or in /usr" + ) os.environ["GDAL_DATA"] = gdal_data os.environ["PROJ_LIB"] = proj_lib -def __find_lib(prefix=None, otb_module=None): +def __find_lib(prefix: str = None, otb_module=None): """Try to find OTB external libraries directory. Args: prefix: try with OTB root directory - otb_module: try with OTB python module (otbApplication) library path if found, else None + otb_module: try with otbApplication library path if found, else None Returns: - lib path + lib path, or None if not found """ if prefix is not None: - lib_dir = prefix / 'lib' + lib_dir = prefix / "lib" if lib_dir.exists(): return lib_dir.absolute() if otb_module is not None: @@ -187,14 +219,14 @@ def __find_lib(prefix=None, otb_module=None): return None -def __find_python_api(lib_dir): +def __find_python_api(lib_dir: Path): """Try to find the python path. Args: prefix: prefix Returns: - python API path if found, else None + OTB python API path, or None if not found """ otb_api = lib_dir / "python" @@ -206,14 +238,14 @@ def __find_python_api(lib_dir): return None -def __find_apps_path(lib_dir): +def __find_apps_path(lib_dir: Path): """Try to find the OTB applications path. Args: lib_dir: library path Returns: - application path if found, else empty string + application path, or empty string if not found """ if lib_dir.exists(): @@ -225,14 +257,11 @@ def __find_apps_path(lib_dir): return "" -def __find_otb_root(scan_userdir=False): +def __find_otb_root(): """Search for OTB root directory in well known locations. - Args: - scan_userdir: search with glob in $HOME directory - Returns: - str path of the OTB directory + str path of the OTB directory, or None if not found """ prefix = None @@ -253,57 +282,72 @@ def __find_otb_root(scan_userdir=False): prefix = path.parent.parent.parent else: prefix = path.parent.parent - prefix = prefix.absolute() elif sys.platform == "win32": - for path in Path("c:/Program Files").glob("**/OTB-*/lib"): + for path in sorted(Path("c:/Program Files").glob("**/OTB-*/lib")): logger.info("Found %s", path.parent) - prefix = path.parent.absolute() - elif sys.platform == "darwin": - # TODO: find OTB in macOS - pass - - # If possible, use OTB found in user's HOME tree (this may take some time) - if scan_userdir: - for path in Path().home().glob("**/OTB-*/lib"): - logger.info("Found %s", path.parent) - prefix = path.parent.absolute() - - return prefix + prefix = path.parent + # Search for pyotb OTB install, or default on macOS + apps = Path.home() / "Applications" + for path in sorted(apps.glob("OTB-*/lib/")): + logger.info("Found %s", path.parent) + prefix = path.parent + # Return latest found prefix (and version), see precedence in find_otb() docstrings + if isinstance(prefix, Path): + return prefix.absolute() + return None -def __suggest_fix_import(error_message, prefix): +def __suggest_fix_import(error_message: str, prefix: str): """Help user to fix the OTB installation with appropriate log messages.""" logger.critical("An error occurred while importing OTB Python API") logger.critical("OTB error message was '%s'", error_message) - if sys.platform == "linux": - if error_message.startswith('libpython3.'): - logger.critical("It seems like you need to symlink or recompile python bindings") - if sys.executable.startswith('/usr/bin'): - lib = f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so" - if which('ctest'): - logger.critical("To recompile python bindings, use 'cd %s ; source otbenv.profile ; " - "ctest -S share/otb/swig/build_wrapping.cmake -VV'", prefix) - elif Path(lib).exists(): - expect_minor = int(error_message[11]) - if expect_minor != sys.version_info.minor: - logger.critical("Python library version mismatch (OTB was expecting 3.%s) : " - "a simple symlink may not work, depending on your python version", expect_minor) - target_lib = f"{prefix}/lib/libpython3.{expect_minor}.so.rh-python3{expect_minor}-1.0" - logger.critical("Use 'ln -s %s %s'", lib, target_lib) - else: - logger.critical("You may need to install cmake in order to recompile python bindings") - else: - logger.critical("Unable to automatically locate python dynamic library of %s", sys.executable) - return - elif sys.platform == "win32": + if sys.platform == "win32": if error_message.startswith("DLL load failed"): if sys.version_info.minor != 7: - logger.critical("You need Python 3.5 (OTB releases 6.4 to 7.4) or Python 3.7 (since OTB 8)") - issue_link = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/2010" - logger.critical("Another workaround is to recompile Python bindings with cmake, see %s", issue_link) + logger.critical( + "You need Python 3.5 (OTB 6.4 to 7.4) or Python 3.7 (since OTB 8)" + ) else: - logger.critical("It seems that your env variables aren't properly set," - " first use 'call otbenv.bat' then try to import pyotb once again") + logger.critical( + "It seems that your env variables aren't properly set," + " first use 'call otbenv.bat' then try to import pyotb once again" + ) + elif error_message.startswith("libpython3."): + logger.critical( + "It seems like you need to symlink or recompile python bindings" + ) + if ( + sys.executable.startswith("/usr/bin") + and which("ctest") + and which("python3-config") + ): + logger.critical( + "To compile, use 'cd %s ; source otbenv.profile ; " + "ctest -S share/otb/swig/build_wrapping.cmake -VV'", + prefix, + ) return - docs_link = "https://www.orfeo-toolbox.org/CookBook/Installation.html" - logger.critical("You can verify installation requirements for your OS at %s", docs_link) + logger.critical( + "You may need to install cmake, python3-dev and mesa's libgl" + " in order to recompile python bindings" + ) + expected = int(error_message[11]) + if expected != sys.version_info.minor: + logger.critical( + "Python library version mismatch (OTB expected 3.%s) : " + "a symlink may not work, depending on your python version", + expected, + ) + lib_dir = sysconfig.get_config_var("LIBDIR") + lib = f"{lib_dir}/libpython3.{sys.version_info.minor}.so" + if Path(lib).exists(): + target = f"{prefix}/lib/libpython3.{expected}.so.1.0" + logger.critical("If using OTB>=8.0, try 'ln -sf %s %s'", lib, target) + logger.critical( + "You can verify installation requirements for your OS at %s", DOCS_URL + ) + + +# This part of pyotb is the first imported during __init__ and checks if OTB is found +# If OTB isn't found, a SystemExit is raised to prevent execution of the core module +find_otb() diff --git a/pyotb/install.py b/pyotb/install.py new file mode 100644 index 0000000000000000000000000000000000000000..0cf139e474495d182dcfb504e99a8b92711f4990 --- /dev/null +++ b/pyotb/install.py @@ -0,0 +1,208 @@ +"""This module contains functions for interactive auto installation of OTB.""" +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys +import sysconfig +import tempfile +import urllib.request +import zipfile +from pathlib import Path +from shutil import which + + +def interactive_config(): + """Prompt user to configure installation variables.""" + version = input("Choose a version to download (default is latest): ") + default_dir = Path.home() / "Applications" + path = input(f"Parent directory for installation (default is {default_dir}): ") + env = input("Permanently change user's environment variables ? (y/n): ") == "y" + return version, path, env + + +def otb_latest_release_tag(): + """Use gitlab API to find latest release tag name, but skip pre-releases.""" + api_endpoint = "https://gitlab.orfeo-toolbox.org/api/v4/projects/53/repository/tags" + vers_regex = re.compile(r"^\d\.\d\.\d$") # we ignore rc-* or alpha-* + with urllib.request.urlopen(api_endpoint) as stream: + data = json.loads(stream.read()) + releases = sorted( + [tag["name"] for tag in data if vers_regex.match(tag["name"])], + ) + return releases[-1] + + +def check_versions(sysname: str, python_minor: int, otb_major: int) -> tuple[bool, int]: + """Verify if python version is compatible with major OTB version. + + Args: + sysname: OTB's system name convention (Linux64, Darwin64, Win64) + python_minor: minor version of python + otb_major: major version of OTB to be installed + + Returns: + (True, 0) if compatible or (False, expected_version) in case of conflicts + + """ + if sysname == "Win64": + expected = 5 if otb_major in (6, 7) else 7 + if python_minor == expected: + return True, 0 + elif sysname == "Darwin64": + expected = 7, 0 + if python_minor == expected: + return True, 0 + elif sysname == "Linux64": + expected = 5 if otb_major in (6, 7) else 8 + if python_minor == expected: + return True, 0 + return False, expected + + +def env_config_unix(otb_path: Path): + """Update env profile for current user with new otb_env.profile call. + + Args: + otb_path: the path of the new OTB installation + + """ + profile = Path.home() / ".profile" + with profile.open("a", encoding="utf-8") as buf: + buf.write(f'\n. "{otb_path}/otbenv.profile"\n') + print(f"##### Added new environment variables to {profile}") + + +def env_config_windows(otb_path: Path): + """Update user's registry hive with new OTB_ROOT env variable. + + Args: + otb_path: path of the new OTB installation + + """ + import winreg # pylint: disable=import-error,import-outside-toplevel + + with winreg.OpenKeyEx( + winreg.HKEY_CURRENT_USER, "Environment", 0, winreg.KEY_SET_VALUE + ) as reg_key: + winreg.SetValueEx(reg_key, "OTB_ROOT", 0, winreg.REG_EXPAND_SZ, str(otb_path)) + print( + "##### Environment variable 'OTB_ROOT' added to user's registry. " + "You'll need to login / logout to apply this change." + ) + reg_cmd = "reg.exe delete HKEY_CURRENT_USER\\Environment /v OTB_ROOT /f" + print(f"To undo this, you may use '{reg_cmd}'") + + +def install_otb(version: str = "latest", path: str = "", edit_env: bool = True): + """Install pre-compiled OTB binaries in path, use latest release by default. + + Args: + version: OTB version tag, e.g. '8.1.2' + path: installation directory, default is $HOME/Applications + edit_env: whether or not to permanently modify user's environment variables + + Returns: + full path of the new installation + + Raises: + SystemExit: if python version is not compatible with major OTB version + SystemError: if automatic env config failed + + """ + # Read env config + if sys.version_info.major == 2: + raise SystemExit("Python 3 is required for OTB bindings.") + python_minor = sys.version_info.minor + if not version or version == "latest": + version = otb_latest_release_tag() + name_corresp = {"linux": "Linux64", "darwin": "Darwin64", "win32": "Win64"} + sysname = name_corresp[sys.platform] + ext = "zip" if sysname == "Win64" else "run" + cmd = which("zsh") or which("bash") or which("sh") + otb_major = int(version[0]) + check, expected = check_versions(sysname, python_minor, otb_major) + if sysname == "Win64" and not check: + raise SystemExit( + f"Python 3.{expected} is required to import bindings on Windows." + ) + + # Fetch archive and run installer + filename = f"OTB-{version}-{sysname}.{ext}" + url = f"https://www.orfeo-toolbox.org/packages/archives/OTB/{filename}" + tmpdir = tempfile.gettempdir() + tmpfile = Path(tmpdir) / filename + print(f"##### Downloading {url}") + urllib.request.urlretrieve(url, tmpfile) + if path: + default_path = False + path = Path(path) + else: + default_path = True + path = Path.home() / "Applications" / tmpfile.stem + if sysname == "Win64": + with zipfile.ZipFile(tmpfile) as zipf: + print("##### Extracting zip file") + # Unzip will always create a dir with OTB-version name + zipf.extractall(path.parent if default_path else path) + else: + install_cmd = f"{cmd} {tmpfile} --target {path} --accept" + print(f"##### Executing '{install_cmd}'\n") + subprocess.run(install_cmd, shell=True, check=True) + tmpfile.unlink() # cleaning + + # Add env variable to profile + if edit_env: + if sysname == "Win64": + env_config_windows(path) + else: + env_config_unix(path) + elif not default_path: + ext = "bat" if sysname == "Win64" else "profile" + print( + f"Remember to call '{path}{os.sep}otbenv.{ext}' before importing pyotb, " + f"or add 'OTB_ROOT=\"{path}\"' to your env variables." + ) + # Requirements are met, no recompilation or symlink required + if check: + return str(path) + + # Else try recompile bindings : can fail because of OpenGL + # Here we check for /usr/bin because CMake's will find_package() only there + if ( + sys.executable.startswith("/usr/bin") + and which("ctest") + and which("python3-config") + ): + try: + print("\n!!!!! Python version mismatch, trying to recompile bindings") + ctest_cmd = ( + ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -V" + ) + print(f"##### Executing '{ctest_cmd}'") + subprocess.run(ctest_cmd, cwd=path, check=True, shell=True) + return str(path) + except subprocess.CalledProcessError: + print("\nCompilation failed.") + # TODO: support for sudo auto build deps install using apt, pacman/yay, brew... + print( + "You need cmake, python3-dev and libgl1-mesa-dev installed." + "\nTrying to symlink libraries instead - this may fail with newest versions." + ) + + # Finally try with cross version python symlink (only tested on Ubuntu) + suffix = "so.1.0" if otb_major >= 8 else f"so.rh-python3{expected}-1.0" + target_lib = f"{path}/lib/libpython3.{expected}.{suffix}" + lib_dir = sysconfig.get_config_var("LIBDIR") + lib = f"{lib_dir}/libpython3.{sys.version_info.minor}.so" + if Path(lib).exists(): + print(f"##### Creating symbolic link: {lib} -> {target_lib}") + ln_cmd = f'ln -sf "{lib}" "{target_lib}"' + subprocess.run(ln_cmd, executable=cmd, shell=True, check=True) + return str(path) + raise SystemError( + f"Unable to automatically locate library for executable '{sys.executable}', " + f"you could manually create a symlink from that file to {target_lib}" + ) diff --git a/pyproject.toml b/pyproject.toml index ddb1465f3a361addd19954bd24367980b66ec754..fdaa0a35167bea61244a4d4cfb500c2bbf7cd7d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,75 @@ [build-system] -requires = [ - "setuptools>=42", - "wheel", - "numpy>=1.13,<1.23" -] +requires = ["setuptools >= 61.0", "wheel"] build-backend = "setuptools.build_meta" + +[project] +name = "pyotb" +description = "Library to enable easy use of the Orfeo ToolBox (OTB) in Python" +authors = [ + { name = "Rémi Cresson", email = "remi.cresson@inrae.fr" }, + { name = "Nicolas Narçon" }, + { name = "Vincent Delbar" }, +] +requires-python = ">=3.7" +keywords = ["gis", "remote sensing", "otb", "orfeotoolbox", "orfeo toolbox"] +dependencies = ["numpy>=1.16"] +readme = "README.md" +license = { text = "Apache-2.0" } +dynamic = ["version"] +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Topic :: Scientific/Engineering :: GIS", + "Topic :: Scientific/Engineering :: Image Processing", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", +] + +[project.optional-dependencies] +dev = [ + "pytest", + "pytest-cov", + "pylint", + "codespell", + "pydocstyle", + "tomli", + "requests", +] + +[project.urls] +documentation = "https://pyotb.readthedocs.io" +homepage = "https://github.com/orfeotoolbox/pyotb" +repository = "https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb" + +[tool.setuptools] +packages = ["pyotb"] + +[tool.setuptools.dynamic] +version = { attr = "pyotb.__version__" } + +[tool.pylint] +max-line-length = 88 +max-module-lines = 2000 +good-names = ["x", "y", "i", "j", "k", "e"] +disable = [ + "line-too-long", + "too-many-locals", + "too-many-branches", + "too-many-statements", + "too-many-instance-attributes", +] + +[tool.pydocstyle] +convention = "google" + +[tool.black] +line-length = 88 + +[tool.pytest.ini_options] +minversion = "7.0" +addopts = "--color=yes --cov=pyotb --no-cov-on-fail --cov-report term" +testpaths = ["tests"] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 778d2740a0ec3dd1d980878d620f444953ee0dee..0000000000000000000000000000000000000000 --- a/setup.cfg +++ /dev/null @@ -1,3 +0,0 @@ -[metadata] -description-file=README.md - diff --git a/setup.py b/setup.py deleted file mode 100644 index 519ba5d8e87a593b9d5caa06e23a39131766b655..0000000000000000000000000000000000000000 --- a/setup.py +++ /dev/null @@ -1,31 +0,0 @@ -# -*- coding: utf-8 -*- -import setuptools - -with open("README.md", "r", encoding="utf-8") as fh: - long_description = fh.read() - -setuptools.setup( - name="pyotb", - version="1.5.4", - author="Nicolas Narçon", - author_email="nicolas.narcon@gmail.com", - description="Library to enable easy use of the Orfeo Tool Box (OTB) in Python", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/", - classifiers=[ - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Topic :: Scientific/Engineering :: GIS", - "Topic :: Scientific/Engineering :: Image Processing", - "License :: OSI Approved :: Apache Software License", - "Operating System :: OS Independent", - ], - packages=setuptools.find_packages(), - python_requires=">=3.6", - keywords="remote sensing, otb, orfeotoolbox, orfeo toolbox", -) -#package_dir={"": "src"}, diff --git a/tests/pipeline_summary.json b/tests/pipeline_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..443f9ef09cf9676debe889a9d775c43924fde596 --- /dev/null +++ b/tests/pipeline_summary.json @@ -0,0 +1,121 @@ +{ + "SIMPLE": { + "name": "ManageNoData", + "parameters": { + "usenan": false, + "mode": "buildmask", + "mode.buildmask.inv": 1.0, + "mode.buildmask.outv": 0.0, + "in": { + "name": "BandMath", + "parameters": { + "il": [ + { + "name": "OrthoRectification", + "parameters": { + "map": "utm", + "map.utm.zone": 31, + "map.utm.northhem": true, + "outputs.mode": "auto", + "outputs.ulx": 560000.8382510637, + "outputs.uly": 5495732.692591702, + "outputs.sizex": 251, + "outputs.sizey": 304, + "outputs.spacingx": 5.997312290795521, + "outputs.spacingy": -5.997312290795521, + "outputs.lrx": 561506.1636360534, + "outputs.lry": 5493909.5096553, + "outputs.isotropic": true, + "interpolator": "bco", + "interpolator.bco.radius": 2, + "opt.rpc": 10, + "opt.gridspacing": 4.0, + "io.in": "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" + } + } + ], + "exp": "im1b1" + } + } + } + }, + "DIAMOND": { + "name": "BandMathX", + "parameters": { + "il": [ + { + "name": "OrthoRectification", + "parameters": { + "map": "utm", + "map.utm.zone": 31, + "map.utm.northhem": true, + "outputs.mode": "auto", + "outputs.ulx": 560000.8382510637, + "outputs.uly": 5495732.692591702, + "outputs.sizex": 251, + "outputs.sizey": 304, + "outputs.spacingx": 5.997312290795521, + "outputs.spacingy": -5.997312290795521, + "outputs.lrx": 561506.1636360534, + "outputs.lry": 5493909.5096553, + "outputs.isotropic": true, + "interpolator": "bco", + "interpolator.bco.radius": 2, + "opt.rpc": 10, + "opt.gridspacing": 4.0, + "io.in": { + "name": "BandMath", + "parameters": { + "il": [ + "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" + ], + "exp": "im1b1" + } + } + } + }, + { + "name": "ManageNoData", + "parameters": { + "usenan": false, + "mode": "buildmask", + "mode.buildmask.inv": 1.0, + "mode.buildmask.outv": 0.0, + "in": { + "name": "OrthoRectification", + "parameters": { + "map": "utm", + "map.utm.zone": 31, + "map.utm.northhem": true, + "outputs.mode": "auto", + "outputs.ulx": 560000.8382510637, + "outputs.uly": 5495732.692591702, + "outputs.sizex": 251, + "outputs.sizey": 304, + "outputs.spacingx": 5.997312290795521, + "outputs.spacingy": -5.997312290795521, + "outputs.lrx": 561506.1636360534, + "outputs.lry": 5493909.5096553, + "outputs.isotropic": true, + "interpolator": "bco", + "interpolator.bco.radius": 2, + "opt.rpc": 10, + "opt.gridspacing": 4.0, + "io.in": { + "name": "BandMath", + "parameters": { + "il": [ + "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" + ], + "exp": "im1b1" + } + } + } + } + } + } + ], + "exp": "im1+im2" + } + } +} diff --git a/tests/test_core.py b/tests/test_core.py index abefc80c7c5a03c06360e900d2e7dabee4e116b3..8b3f8fae5fbe66124ceb3c6911e09aca11ba010b 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1,88 +1,486 @@ -import os -import pyotb -from ast import literal_eval -from pathlib import Path +import pytest +import numpy as np +from tests_data import * -FILEPATH = os.environ["TEST_INPUT_IMAGE"] -INPUT = pyotb.Input(FILEPATH) +def test_app_parameters(): + # Input / ExtractROI + assert INPUT.parameters + assert (INPUT.parameters["sizex"], INPUT.parameters["sizey"]) == (251, 304) + # OrthoRectification + app = pyotb.OrthoRectification(INPUT) + assert isinstance(app.parameters["map"], str) + assert app.parameters["map"] == "utm" + assert "map" in app._auto_parameters + app.set_parameters({"map": "epsg", "map.epsg.code": 2154}) + assert app.parameters["map"] == "epsg" + assert "map" in app._settings and "map" not in app._auto_parameters + assert app.parameters["map.epsg.code"] == app.app.GetParameters()["map.epsg.code"] + # Orthorectification with underscore kwargs + app = pyotb.OrthoRectification(io_in=INPUT, map_epsg_code=2154) + assert app.parameters["map.epsg.code"] == 2154 + # ManageNoData + app = pyotb.ManageNoData(INPUT) + assert "usenan" in app._auto_parameters + assert "mode.buildmask.inv" in app._auto_parameters + # OpticalCalibration + app = pyotb.OpticalCalibration(pyotb.Input(PLEIADES_IMG_URL), level="toa") + assert "milli" in app._auto_parameters + assert "clamp" in app._auto_parameters + assert app._auto_parameters["acqui.year"] == 2012 + assert app._auto_parameters["acqui.sun.elev"] == 23.836299896240234 -# Basic tests -def test_dtype(): - assert INPUT.dtype == "uint8" + +def test_app_properties(): + assert INPUT.input_key == INPUT.input_image_key == "in" + assert INPUT.output_key == INPUT.output_image_key == "out" + with pytest.raises(KeyError): + pyotb.BandMath(INPUT, expression="im1b1") + # Test user can set custom name + app = pyotb.App("BandMath", [INPUT], exp="im1b1", name="TestName") + assert app.name == "TestName" + # Test data dict is not empty + app = pyotb.ReadImageInfo(INPUT) + assert app.data + # Test elapsed time is not null + assert 0 < app.elapsed_time < 1 -def test_shape(): +def test_app_input_vsi(): + # Ensure old way is still working: ExtractROI will raise RuntimeError if a path is malformed + pyotb.Input("/vsicurl/" + SPOT_IMG_URL) + # Simple remote file + info = pyotb.ReadImageInfo("https://fake.com/image.tif", frozen=True) + assert ( + info.app.GetParameterValue("in") + == info.parameters["in"] + == "/vsicurl/https://fake.com/image.tif" + ) + # Compressed single file archive + info = pyotb.ReadImageInfo("image.tif.zip", frozen=True) + assert ( + info.app.GetParameterValue("in") + == info.parameters["in"] + == "/vsizip/image.tif.zip" + ) + # File within compressed remote archive + info = pyotb.ReadImageInfo("https://fake.com/archive.tar.gz/image.tif", frozen=True) + assert ( + info.app.GetParameterValue("in") + == info.parameters["in"] + == "/vsitar//vsicurl/https://fake.com/archive.tar.gz/image.tif" + ) + # Piped curl --> zip --> tiff + ziped_tif_urls = ( + "https://github.com/OSGeo/gdal/raw/master" + "/autotest/gcore/data/byte.tif.zip", # without /vsi + "/vsizip/vsicurl/https://github.com/OSGeo/gdal/raw/master" + "/autotest/gcore/data/byte.tif.zip", # with /vsi + ) + for ziped_tif_url in ziped_tif_urls: + info = pyotb.ReadImageInfo(ziped_tif_url) + assert info["sizex"] == 20 + + +def test_img_properties(): + assert INPUT.dtype == "uint8" assert INPUT.shape == (304, 251, 4) + assert INPUT.transform == (6.0, 0.0, 760056.0, 0.0, -6.0, 6946092.0) + with pytest.raises(TypeError): + assert pyotb.ReadImageInfo(INPUT).dtype == "uint8" + + +def test_img_metadata(): + assert "ProjectionRef" in INPUT.metadata + assert "TIFFTAG_SOFTWARE" in INPUT.metadata + inp2 = pyotb.Input( + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/" + "47/Q/RU/2021/12/S2B_47QRU_20211227_0_L2A/B04.tif" + ) + assert "ProjectionRef" in inp2.metadata + assert "OVR_RESAMPLING_ALG" in inp2.metadata + # Metadata with numeric values (e.g. TileHintX) + fp = ( + "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/" + "Data/Input/radarsat2/RADARSAT2_ALTONA_300_300_VV.tif?inline=false" + ) + app = pyotb.BandMath({"il": [fp], "exp": "im1b1"}) + assert "TileHintX" in app.metadata + + +def test_essential_apps(): + readimageinfo = pyotb.ReadImageInfo(INPUT, quiet=True) + assert (readimageinfo["sizex"], readimageinfo["sizey"]) == (251, 304) + assert readimageinfo["numberbands"] == 4 + computeimagestats = pyotb.ComputeImagesStatistics([INPUT], quiet=True) + assert computeimagestats["out.min"] == TEST_IMAGE_STATS["out.min"] + slicer_computeimagestats = pyotb.ComputeImagesStatistics( + il=[INPUT[:10, :10, 0]], quiet=True + ) + assert slicer_computeimagestats["out.min"] == [180] + + +def test_get_statistics(): + stats_data = pyotb.ComputeImagesStatistics(INPUT).data + assert stats_data == TEST_IMAGE_STATS + assert INPUT.get_statistics() == TEST_IMAGE_STATS + + +def test_get_info(): + infos = INPUT.get_info() + assert (infos["sizex"], infos["sizey"]) == (251, 304) + bm_infos = pyotb.BandMathX([INPUT], exp="im1")["out"].get_info() + assert infos == bm_infos + + +def test_read_values_at_coords(): + assert INPUT[0, 0, 0] == 180 + assert INPUT[10, 20, :] == [207, 192, 172, 255] + + +def test_xy_to_rowcol(): + assert INPUT.get_rowcol_from_xy(760101, 6945977) == (19, 7) + + +def test_write(): + assert INPUT.write("/dev/shm/test_write.tif", ext_fname="nodata=0") + INPUT["out"].filepath.unlink() + assert INPUT.write(Path("/dev/shm/test_write.tif"), ext_fname="nodata=0") + INPUT["out"].filepath.unlink() + # Frozen + frozen_app = pyotb.BandMath(INPUT, exp="im1b1", frozen=True) + assert frozen_app.write("/dev/shm/test_frozen_app_write.tif") + frozen_app["out"].filepath.unlink() + frozen_app_init_with_outfile = pyotb.BandMath( + INPUT, exp="im1b1", out="/dev/shm/test_frozen_app_write.tif", frozen=True + ) + assert frozen_app_init_with_outfile.write(pixel_type="uint16") + assert frozen_app_init_with_outfile.dtype == "uint16" + frozen_app_init_with_outfile["out"].filepath.unlink() + + +def test_write_multi_output(): + mss = pyotb.MeanShiftSmoothing( + SPOT_IMG_URL, + fout="/dev/shm/test_ext_fn_fout.tif", + foutpos="/dev/shm/test_ext_fn_foutpos.tif", + ) + + mss = pyotb.MeanShiftSmoothing(SPOT_IMG_URL) + assert mss.write( + { + "fout": "/dev/shm/test_ext_fn_fout.tif", + "foutpos": "/dev/shm/test_ext_fn_foutpos.tif", + }, + ext_fname={"nodata": 0, "gdal:co:COMPRESS": "DEFLATE"}, + ) + + dr = pyotb.DimensionalityReduction( + SPOT_IMG_URL, out="/dev/shm/1.tif", outinv="/dev/shm/2.tif" + ) + dr = pyotb.DimensionalityReduction(SPOT_IMG_URL) + assert dr.write( + {"out": "/dev/shm/1.tif", "outinv": "/dev/shm/2.tif"} + ) + + +def test_write_ext_fname(): + def _check(expected: str, key: str = "out", app=INPUT.app): + fn = app.GetParameterString(key) + assert "?&" in fn + assert fn.split("?&", 1)[1] == expected + + assert INPUT.write("/dev/shm/test_write.tif", ext_fname="nodata=0") + _check("nodata=0") + assert INPUT.write("/dev/shm/test_write.tif", ext_fname={"nodata": "0"}) + _check("nodata=0") + assert INPUT.write("/dev/shm/test_write.tif", ext_fname={"nodata": 0}) + _check("nodata=0") + assert INPUT.write( + "/dev/shm/test_write.tif", + ext_fname={"nodata": 0, "gdal:co:COMPRESS": "DEFLATE"}, + ) + _check("nodata=0&gdal:co:COMPRESS=DEFLATE") + assert INPUT.write( + "/dev/shm/test_write.tif", ext_fname="nodata=0&gdal:co:COMPRESS=DEFLATE" + ) + _check("nodata=0&gdal:co:COMPRESS=DEFLATE") + assert INPUT.write( + "/dev/shm/test_write.tif?&box=0:0:10:10", + ext_fname={"nodata": "0", "gdal:co:COMPRESS": "DEFLATE", "box": "0:0:20:20"}, + ) + # Check that the bbox is the one specified in the filepath, not the one + # specified in `ext_filename` + _check("nodata=0&gdal:co:COMPRESS=DEFLATE&box=0:0:10:10") + assert INPUT.write( + "/dev/shm/test_write.tif?&box=0:0:10:10", + ext_fname="nodata=0&gdal:co:COMPRESS=DEFLATE&box=0:0:20:20", + ) + _check("nodata=0&gdal:co:COMPRESS=DEFLATE&box=0:0:10:10") + INPUT["out"].filepath.unlink() + + mmsd = pyotb.MorphologicalMultiScaleDecomposition(INPUT) + mmsd.write( + { + "outconvex": "/dev/shm/outconvex.tif?&nodata=1", + "outconcave": "/dev/shm/outconcave.tif?&nodata=2", + "outleveling": "/dev/shm/outleveling.tif?&nodata=3", + }, + ext_fname={"nodata": 0, "gdal:co:COMPRESS": "DEFLATE"}, + ) + _check("nodata=1&gdal:co:COMPRESS=DEFLATE", key="outconvex", app=mmsd.app) + _check("nodata=2&gdal:co:COMPRESS=DEFLATE", key="outconcave", app=mmsd.app) + _check("nodata=3&gdal:co:COMPRESS=DEFLATE", key="outleveling", app=mmsd.app) + mmsd["outconvex"].filepath.unlink() + mmsd["outconcave"].filepath.unlink() + mmsd["outleveling"].filepath.unlink() -def test_slicer_shape(): - extract = INPUT[:50, :60, :3] - assert extract.shape == (50, 60, 3) +def test_output(): + assert INPUT["out"].write("/dev/shm/test_output_write.tif") + INPUT["out"].filepath.unlink() + frozen_app = pyotb.BandMath(INPUT, exp="im1b1", frozen=True) + assert frozen_app["out"].write("/dev/shm/test_frozen_app_write.tif") + frozen_app["out"].filepath.unlink() + info_from_output_obj = pyotb.ReadImageInfo(INPUT["out"]) + assert info_from_output_obj.data -def test_slicer_preserve_dtype(): - extract = INPUT[:50, :60, :3] - assert extract.dtype == "uint8" +# Slicer +def test_slicer(): + sliced = INPUT[:50, :60, :3] + assert sliced.parameters["cl"] == ["Channel1", "Channel2", "Channel3"] + assert sliced.shape == (50, 60, 3) + assert sliced.dtype == "uint8" + sliced_negative_band_idx = INPUT[:50, :60, :-2] + assert sliced_negative_band_idx.shape == (50, 60, 2) + sliced_from_output = pyotb.BandMath([INPUT], exp="im1b1")["out"][:50, :60, :-2] + assert isinstance(sliced_from_output, pyotb.core.Slicer) -# More complex tests -def test_operation(): +# Operation and LogicalOperation +def test_operator_expressions(): op = INPUT / 255 * 128 - assert op.exp == "((im1b1 / 255) * 128);((im1b2 / 255) * 128);((im1b3 / 255) * 128);((im1b4 / 255) * 128)" + assert ( + op.exp + == "((im1b1 / 255) * 128);((im1b2 / 255) * 128);((im1b3 / 255) * 128);((im1b4 / 255) * 128)" + ) + assert op.dtype == "float32" + assert abs(INPUT).exp == "(abs(im1b1));(abs(im1b2));(abs(im1b3));(abs(im1b4))" + summed_bands = sum(INPUT[:, :, b] for b in range(INPUT.shape[-1])) + assert summed_bands.exp == "((((0 + im1b1) + im1b2) + im1b3) + im1b4)" -def test_sum_bands(): - # Sum of bands - summed = sum(INPUT[:, :, b] for b in range(INPUT.shape[-1])) - assert summed.exp == "((((0 + im1b1) + im1b2) + im1b3) + im1b4)" +def operation_test(func, exp): + meas = func(INPUT) + ref = pyotb.BandMathX({"il": [SPOT_IMG_URL], "exp": exp}) + for i in range(1, 5): + compared = pyotb.CompareImages( + {"ref.in": ref, "meas.in": meas, "ref.channel": i, "meas.channel": i} + ) + assert (compared["count"], compared["mse"]) == (0, 0) -def test_binary_mask_where(): - # Create binary mask based on several possible values - values = [1, 2, 3, 4] - res = pyotb.where(pyotb.any(INPUT[:, :, 0] == value for value in values), 255, 0) - assert res.exp == "(((((im1b1 == 1) || (im1b1 == 2)) || (im1b1 == 3)) || (im1b1 == 4)) ? 255 : 0)" +def test_operation_add(): + operation_test(lambda x: x + x, "im1 + im1") + operation_test(lambda x: x + INPUT, "im1 + im1") + operation_test(lambda x: INPUT + x, "im1 + im1") + operation_test(lambda x: x + 2, "im1 + {2;2;2;2}") + operation_test(lambda x: x + 2.0, "im1 + {2.0;2.0;2.0;2.0}") + operation_test(lambda x: 2 + x, "{2;2;2;2} + im1") + operation_test(lambda x: 2.0 + x, "{2.0;2.0;2.0;2.0} + im1") -# Apps -def test_app_readimageinfo(): - info = pyotb.ReadImageInfo(INPUT, quiet=True) - assert info.sizex == 251 - assert info.sizey == 304 - assert info["numberbands"] == info.numberbands == 4 +def test_operation_sub(): + operation_test(lambda x: x - x, "im1 - im1") + operation_test(lambda x: x - INPUT, "im1 - im1") + operation_test(lambda x: INPUT - x, "im1 - im1") + operation_test(lambda x: x - 2, "im1 - {2;2;2;2}") + operation_test(lambda x: x - 2.0, "im1 - {2.0;2.0;2.0;2.0}") + operation_test(lambda x: 2 - x, "{2;2;2;2} - im1") + operation_test(lambda x: 2.0 - x, "{2.0;2.0;2.0;2.0} - im1") -def test_app_computeimagestats(): - stats = pyotb.ComputeImagesStatistics([INPUT], quiet=True) - assert stats["out.min"] == "[33, 64, 91, 47]" +def test_operation_mult(): + operation_test(lambda x: x * x, "im1 mult im1") + operation_test(lambda x: x * INPUT, "im1 mult im1") + operation_test(lambda x: INPUT * x, "im1 mult im1") + operation_test(lambda x: x * 2, "im1 * 2") + operation_test(lambda x: x * 2.0, "im1 * 2.0") + operation_test(lambda x: 2 * x, "2 * im1") + operation_test(lambda x: 2.0 * x, "2.0 * im1") -def test_app_computeimagestats_sliced(): - slicer_stats = pyotb.ComputeImagesStatistics(il=[INPUT[:10, :10, 0]], quiet=True) - assert slicer_stats["out.min"] == "[180]" +def test_operation_div(): + operation_test(lambda x: x / x, "im1 div im1") + operation_test(lambda x: x / INPUT, "im1 div im1") + operation_test(lambda x: INPUT / x, "im1 div im1") + operation_test(lambda x: x / 2, "im1 * 0.5") + operation_test(lambda x: x / 2.0, "im1 * 0.5") + operation_test(lambda x: 2 / x, "{2;2;2;2} div im1") + operation_test(lambda x: 2.0 / x, "{2.0;2.0;2.0;2.0} div im1") -# NDVI +# BandMath NDVI == RadiometricIndices NDVI ? def test_ndvi_comparison(): - ndvi_bandmath = (INPUT[:, :, -1] - INPUT[:, :, [0]]) / (INPUT[:, :, -1] + INPUT[:, :, 0]) + ndvi_bandmath = (INPUT[:, :, -1] - INPUT[:, :, [0]]) / ( + INPUT[:, :, -1] + INPUT[:, :, 0] + ) ndvi_indices = pyotb.RadiometricIndices( - {"in": INPUT, "list": "Vegetation:NDVI", "channels.red": 1, "channels.nir": 4} + INPUT, {"list": ["Vegetation:NDVI"], "channels.red": 1, "channels.nir": 4} ) assert ndvi_bandmath.exp == "((im1b4 - im1b1) / (im1b4 + im1b1))" + assert ndvi_bandmath.write("/dev/shm/ndvi_bandmath.tif", "float") + assert ndvi_indices.write("/dev/shm/ndvi_indices.tif", "float") - ndvi_bandmath.write("/tmp/ndvi_bandmath.tif", pixel_type="float") - assert Path("/tmp/ndvi_bandmath.tif").exists() - ndvi_indices.write("/tmp/ndvi_indices.tif", pixel_type="float") - assert Path("/tmp/ndvi_indices.tif").exists() + compared = pyotb.CompareImages( + {"ref.in": ndvi_indices, "meas.in": "/dev/shm/ndvi_bandmath.tif"} + ) + assert (compared["count"], compared["mse"]) == (0, 0) + thresholded_indices = pyotb.where(ndvi_indices >= 0.3, 1, 0) + assert thresholded_indices["exp"] == "((im1b1 >= 0.3) ? 1 : 0)" + thresholded_bandmath = pyotb.where(ndvi_bandmath >= 0.3, 1, 0) + assert ( + thresholded_bandmath["exp"] + == "((((im1b4 - im1b1) / (im1b4 + im1b1)) >= 0.3) ? 1 : 0)" + ) - compared = pyotb.CompareImages({"ref.in": ndvi_indices, "meas.in": "/tmp/ndvi_bandmath.tif"}) - assert compared.count == 0 - assert compared.mse == 0 - thresholded_indices = pyotb.where(ndvi_indices >= 0.3, 1, 0) - assert thresholded_indices.exp == "((im1b1 >= 0.3) ? 1 : 0)" +# Tests for functions.py +def test_binary_mask_where(): + # Create binary mask based on several possible values + values = [1, 2, 3, 4] + res = pyotb.where(pyotb.any(INPUT[:, :, 0] == value for value in values), 255, 0) + assert ( + res.exp + == "(((((im1b1 == 1) || (im1b1 == 2)) || (im1b1 == 3)) || (im1b1 == 4)) ? 255 : 0)" + ) - thresholded_bandmath = pyotb.where(ndvi_bandmath >= 0.3, 1, 0) - assert thresholded_bandmath.exp == "((((im1b4 - im1b1) / (im1b4 + im1b1)) >= 0.3) ? 1 : 0)" + +# Tests for summarize() +def test_summarize_pipeline_simple(): + app1 = pyotb.OrthoRectification({"io.in": SPOT_IMG_URL}) + app2 = pyotb.BandMath({"il": [app1], "exp": "im1b1"}) + app3 = pyotb.ManageNoData({"in": app2}) + summary = pyotb.summarize(app3) + assert SIMPLE_SERIALIZATION == summary + + +def test_summarize_pipeline_diamond(): + app1 = pyotb.BandMath({"il": [SPOT_IMG_URL], "exp": "im1b1"}) + app2 = pyotb.OrthoRectification({"io.in": app1}) + app3 = pyotb.ManageNoData({"in": app2}) + app4 = pyotb.BandMathX({"il": [app2, app3], "exp": "im1+im2"}) + summary = pyotb.summarize(app4) + assert DIAMOND_SERIALIZATION == summary + + +def test_summarize_output_obj(): + assert pyotb.summarize(INPUT["out"]) + + +def test_summarize_strip_output(): + in_fn = "/vsicurl/" + SPOT_IMG_URL + in_fn_w_ext = "/vsicurl/" + SPOT_IMG_URL + "?&skipcarto=1" + out_fn = "/dev/shm/output.tif" + out_fn_w_ext = out_fn + "?&box=10:10:10:10" + + baseline = [ + (in_fn, out_fn_w_ext, "out", {}, out_fn_w_ext), + (in_fn, out_fn_w_ext, "out", {"strip_outpath": True}, out_fn), + (in_fn_w_ext, out_fn, "in", {}, in_fn_w_ext), + (in_fn_w_ext, out_fn, "in", {"strip_inpath": True}, in_fn), + ] + + for inp, out, key, extra_args, expected in baseline: + app = pyotb.ExtractROI({"in": inp, "out": out}) + summary = pyotb.summarize(app, **extra_args) + assert ( + summary["parameters"][key] == expected + ), f"Failed for input {inp}, output {out}, args {extra_args}" + + +def test_summarize_consistency(): + app_fns = [ + lambda inp: pyotb.ExtractROI( + {"in": inp, "startx": 10, "starty": 10, "sizex": 50, "sizey": 50} + ), + lambda inp: pyotb.ManageNoData({"in": inp, "mode": "changevalue"}), + lambda inp: pyotb.DynamicConvert({"in": inp}), + lambda inp: pyotb.Mosaic({"il": [inp]}), + lambda inp: pyotb.BandMath({"il": [inp], "exp": "im1b1 + 1"}), + lambda inp: pyotb.BandMathX({"il": [inp], "exp": "im1"}), + lambda inp: pyotb.OrthoRectification({"io.in": inp}), + ] + + def operator_test(app_fn): + """ + Here we create 2 summaries: + - summary of the app before write() + - summary of the app after write() + Then we check that both only differ with the output parameter + """ + app = app_fn(inp=SPOT_IMG_URL) + out_file = "/dev/shm/out.tif" + out_key = app.output_image_key + summary_wo_wrt = pyotb.summarize(app) + app.write(out_file) + summay_w_wrt = pyotb.summarize(app) + app[out_key].filepath.unlink() + summary_wo_wrt["parameters"].update({out_key: out_file}) + assert summary_wo_wrt == summay_w_wrt + + for app_fn in app_fns: + operator_test(app_fn) + + +# Numpy tests +def test_numpy_exports_dic(): + INPUT.export() + exported_array = INPUT.exports_dic[INPUT.output_image_key]["array"] + assert isinstance(exported_array, np.ndarray) + assert exported_array.dtype == "uint8" + del INPUT.exports_dic["out"] + INPUT["out"].export() + assert INPUT["out"].output_image_key in INPUT["out"].exports_dic + + +def test_numpy_conversions(): + array = INPUT.to_numpy() + assert array.dtype == np.uint8 + assert array.shape == INPUT.shape + assert (array.min(), array.max()) == (33, 255) + # Sliced img to array + sliced = INPUT[:100, :200, :3] + sliced_array = sliced.to_numpy() + assert sliced_array.dtype == np.uint8 + assert sliced_array.shape == (100, 200, 3) + # Test auto convert to numpy + assert isinstance(np.array(INPUT), np.ndarray) + assert INPUT.shape == np.array(INPUT).shape + assert INPUT[19, 7] == list(INPUT.to_numpy()[19, 7]) + # Add noise test from the docs + white_noise = np.random.normal(0, 50, size=INPUT.shape) + noisy_image = INPUT + white_noise + assert isinstance(noisy_image, pyotb.core.App) + assert noisy_image.shape == INPUT.shape + + +def test_numpy_to_rasterio(): + array, profile = INPUT.to_rasterio() + assert array.dtype == profile["dtype"] == np.uint8 + assert array.shape == (4, 304, 251) + assert profile["transform"] == (6.0, 0.0, 760056.0, 0.0, -6.0, 6946092.0) + + # CRS test requires GDAL python bindings + try: + from osgeo import osr + + crs = osr.SpatialReference() + crs.ImportFromEPSG(2154) + dest_crs = osr.SpatialReference() + dest_crs.ImportFromWkt(profile["crs"]) + assert dest_crs.IsSame(crs) + except ImportError: + pass diff --git a/tests/test_numpy.py b/tests/test_numpy.py deleted file mode 100644 index 5b1dd048cb73d2a9c904c3b937c6d60d8b8520c3..0000000000000000000000000000000000000000 --- a/tests/test_numpy.py +++ /dev/null @@ -1,53 +0,0 @@ -import os -import numpy as np -import pyotb - - -FILEPATH = os.environ["TEST_INPUT_IMAGE"] -INPUT = pyotb.Input(FILEPATH) - - -def test_to_numpy(): - array = INPUT.to_numpy() - assert array.dtype == np.uint8 - assert array.shape == INPUT.shape - assert array.min() == 33 - assert array.max() == 255 - - -def test_to_numpy_sliced(): - sliced = INPUT[:100, :200, :3] - array = sliced.to_numpy() - assert array.dtype == np.uint8 - assert array.shape == (100, 200, 3) - - -def test_convert_to_array(): - array = np.array(INPUT) - assert isinstance(array, np.ndarray) - assert INPUT.shape == array.shape - - -def test_add_noise_array(): - white_noise = np.random.normal(0, 50, size=INPUT.shape) - noisy_image = INPUT + white_noise - assert isinstance(noisy_image, pyotb.otbObject) - assert noisy_image.shape == INPUT.shape - - -def test_to_rasterio(): - array, profile = INPUT.to_rasterio() - assert array.dtype == profile["dtype"] == np.uint8 - assert array.shape == (4, 304, 251) - assert profile["transform"] == (6.0, 0.0, 760056.0, 0.0, -6.0, 6946092.0) - - # CRS test requires GDAL python bindings - try: - from osgeo import osr - crs = osr.SpatialReference() - crs.ImportFromEPSG(2154) - dest_crs = osr.SpatialReference() - dest_crs.ImportFromWkt(profile["crs"]) - assert dest_crs.IsSame(crs) - except ImportError: - pass diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index c2d8f0e910851f5412f044c5d4ef887613af57a0..5646674e483c04767098619d2e156586d9c1d2d7 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -1,9 +1,8 @@ -import sys import os import itertools import pytest import pyotb -from pyotb.helpers import logger +from tests_data import INPUT, SPOT_IMG_URL # List of buildings blocks, we can add other pyotb objects here @@ -13,18 +12,16 @@ OTBAPPS_BLOCKS = [ lambda inp: pyotb.DynamicConvert({"in": inp}), lambda inp: pyotb.Mosaic({"il": [inp]}), lambda inp: pyotb.BandMath({"il": [inp], "exp": "im1b1 + 1"}), - lambda inp: pyotb.BandMathX({"il": [inp], "exp": "im1"}) + lambda inp: pyotb.BandMathX({"il": [inp], "exp": "im1"}), ] PYOTB_BLOCKS = [ - lambda inp: 1 + abs(inp) * 2, + lambda inp: 1 / (1 + abs(inp) * 2), lambda inp: inp[:80, 10:60, :], ] PIPELINES_LENGTH = [1, 2, 3] ALL_BLOCKS = PYOTB_BLOCKS + OTBAPPS_BLOCKS -FILEPATH = os.environ["TEST_INPUT_IMAGE"] -INPUT = pyotb.Input(FILEPATH) def generate_pipeline(inp, building_blocks): @@ -77,14 +74,18 @@ def pipeline2str(pipeline): a string """ - return " > ".join([INPUT.__class__.__name__] + [f"{i}.{app.name.split()[0]}" - for i, app in enumerate(pipeline)]) + return " > ".join( + [INPUT.__class__.__name__] + + [f"{i}.{app.name.split()[0]}" for i, app in enumerate(pipeline)] + ) def make_pipelines_list(): """Create a list of pipelines using different lengths and blocks""" - blocks = {FILEPATH: OTBAPPS_BLOCKS, # for filepath, we can't use Slicer or Operation - INPUT: ALL_BLOCKS} + blocks = { + SPOT_IMG_URL: OTBAPPS_BLOCKS, + INPUT: ALL_BLOCKS, + } # for filepath, we can't use Slicer or Operation pipelines = [] names = [] for inp, blocks in blocks.items(): @@ -101,53 +102,83 @@ def make_pipelines_list(): return pipelines, names -PIPELINES, NAMES = make_pipelines_list() - - -@pytest.mark.parametrize("pipe", PIPELINES, ids=NAMES) -def test_pipeline_shape(pipe): - for i, app in enumerate(pipe): - print(app.shape) - assert bool(app.shape) +def shape(pipe): + for app in pipe: + yield bool(app.shape) -@pytest.mark.parametrize("pipe", PIPELINES, ids=NAMES) -def test_pipeline_shape_nointermediate(pipe): +def shape_nointermediate(pipe): app = [pipe[-1]][0] - assert bool(app.shape) + yield bool(app.shape) -@pytest.mark.parametrize("pipe", PIPELINES, ids=NAMES) -def test_pipeline_shape_backward(pipe): - for i, app in enumerate(reversed(pipe)): - assert bool(app.shape) +def shape_backward(pipe): + for app in reversed(pipe): + yield bool(app.shape) -@pytest.mark.parametrize("pipe", PIPELINES, ids=NAMES) -def test_pipeline_write(pipe): +def write(pipe): for i, app in enumerate(pipe): out = f"/tmp/out_{i}.tif" if os.path.isfile(out): os.remove(out) - app.write(out) - assert os.path.isfile(out) + yield app.write(out) -@pytest.mark.parametrize("pipe", PIPELINES, ids=NAMES) -def test_pipeline_write_nointermediate(pipe): +def write_nointermediate(pipe): app = [pipe[-1]][0] - out = f"/tmp/out_0.tif" + out = "/tmp/out_0.tif" if os.path.isfile(out): os.remove(out) - app.write(out) - assert os.path.isfile(out) + yield app.write(out) -@pytest.mark.parametrize("pipe", PIPELINES, ids=NAMES) -def test_pipeline_write_backward(pipe): +def write_backward(pipe): for i, app in enumerate(reversed(pipe)): out = f"/tmp/out_{i}.tif" if os.path.isfile(out): os.remove(out) - app.write(out) - assert os.path.isfile(out) + yield app.write(out) + + +funcs = [ + shape, + shape_nointermediate, + shape_backward, + write, + write_nointermediate, + write_backward, +] + +PIPELINES, NAMES = make_pipelines_list() + + +@pytest.mark.parametrize("test_func", funcs) +def test(test_func): + fname = test_func.__name__ + successes, failures = 0, 0 + total_successes = [] + for pipeline, blocks in zip(PIPELINES, NAMES): + err = None + try: + # Here we count non-empty shapes or write results, no errors during exec + bool_tests = list(test_func(pipeline)) + overall_success = all(bool_tests) + except Exception as e: + # Unexpected exception in the pipeline, e.g. a RuntimeError we missed + bool_tests = [] + overall_success = False + err = e + if overall_success: + print(f"\033[92m{fname}: success with [{blocks}]\033[0m\n") + successes += 1 + else: + print(f"\033[91m{fname}: failure with [{blocks}]\033[0m {bool_tests}\n") + if err: + print(f"exception thrown: {err}") + failures += 1 + total_successes.append(overall_success) + print(f"\nEnded test {fname} with {successes} successes, {failures} failures") + if err: + raise err + assert all(total_successes) diff --git a/tests/test_serialization.py b/tests/test_serialization.py deleted file mode 100644 index 2e0ce7fabafaeaab49fae5839eeb9415b38f2f6c..0000000000000000000000000000000000000000 --- a/tests/test_serialization.py +++ /dev/null @@ -1,42 +0,0 @@ -import os -import pyotb - -filepath = os.environ["TEST_INPUT_IMAGE"] - - -def test_pipeline_simple(): - # BandMath -> OrthoRectification -> ManageNoData - app1 = pyotb.BandMath({'il': [filepath], 'exp': 'im1b1'}) - app2 = pyotb.OrthoRectification({'io.in': app1}) - app3 = pyotb.ManageNoData({'in': app2}) - summary = app3.summarize() - reference = {'name': 'ManageNoData', 'parameters': {'in': { - 'name': 'OrthoRectification', 'parameters': {'io.in': { - 'name': 'BandMath', 'parameters': {'il': ('tests/image.tif',), 'exp': 'im1b1'}}, - 'map': 'utm', - 'outputs.isotropic': True}}, - 'mode': 'buildmask'}} - assert summary == reference - - -def test_pipeline_diamond(): - # Diamond graph - app1 = pyotb.BandMath({'il': [filepath], 'exp': 'im1b1'}) - app2 = pyotb.OrthoRectification({'io.in': app1}) - app3 = pyotb.ManageNoData({'in': app2}) - app4 = pyotb.BandMathX({'il': [app2, app3], 'exp': 'im1+im2'}) - summary = app4.summarize() - reference = {'name': 'BandMathX', 'parameters': {'il': [ - {'name': 'OrthoRectification', 'parameters': {'io.in': { - 'name': 'BandMath', 'parameters': {'il': ('tests/image.tif',), 'exp': 'im1b1'}}, - 'map': 'utm', - 'outputs.isotropic': True}}, - {'name': 'ManageNoData', 'parameters': {'in': { - 'name': 'OrthoRectification', 'parameters': { - 'io.in': {'name': 'BandMath', 'parameters': {'il': ('tests/image.tif',), 'exp': 'im1b1'}}, - 'map': 'utm', - 'outputs.isotropic': True}}, - 'mode': 'buildmask'}} - ], - 'exp': 'im1+im2'}} - assert summary == reference diff --git a/tests/tests_data.py b/tests/tests_data.py new file mode 100644 index 0000000000000000000000000000000000000000..2e40c95e1c1b493b1382bb723b3ed12c28ee35e8 --- /dev/null +++ b/tests/tests_data.py @@ -0,0 +1,22 @@ +import json +from pathlib import Path +import pyotb + + +SPOT_IMG_URL = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" +PLEIADES_IMG_URL = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Baseline/OTB/Images/prTvOrthoRectification_pleiades-1_noDEM.tif" +INPUT = pyotb.Input(SPOT_IMG_URL) + + +TEST_IMAGE_STATS = { + "out.mean": [79.5505, 109.225, 115.456, 249.349], + "out.min": [33, 64, 91, 47], + "out.max": [255, 255, 230, 255], + "out.std": [51.0754, 35.3152, 23.4514, 20.3827], +} + +json_file = Path(__file__).parent / "pipeline_summary.json" +with json_file.open("r", encoding="utf-8") as js: + data = json.load(js) +SIMPLE_SERIALIZATION = data["SIMPLE"] +DIAMOND_SERIALIZATION = data["DIAMOND"]