diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5cfe0b9eef546c2b4f0c7be595f7c15d394dbae6..798ece96a05534e95653f82f2f80477fdf1109a1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: gitlab-registry.irstea.fr/remi.cresson/otbtf:3.1-cpu-basic-testing +image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME variables: OTB_BUILD: /src/otb/build/OTB/build # Local OTB build directory @@ -10,104 +10,116 @@ variables: workflow: rules: - if: $CI_MERGE_REQUEST_ID # Execute jobs in merge request context - - if: $CI_COMMIT_BRANCH == 'develop' # Execute jobs when a new commit is pushed to develop branch - + stages: - Build - Static Analysis - Test - Applications Test + - Ship -.update_otbtf_src: &update_otbtf_src - - sudo rm -rf $OTBTF_SRC && sudo ln -s $PWD $OTBTF_SRC # Replace local OTBTF source directory - -.compile_otbtf: &compile_otbtf - - cd $OTB_BUILD && sudo make install -j$(nproc --all) # Rebuild OTB with new OTBTF sources - -.install_pytest: &install_pytest - - pip3 install pytest pytest-cov pytest-order # Install pytest stuff - -before_script: - - *update_otbtf_src - -build: +test docker image: stage: Build allow_failure: false + tags: [godzilla] + image: docker/compose:latest + except: + - master + services: + - name: docker:dind + before_script: + - echo -n $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY + timeout: 10 hours script: - - *compile_otbtf + - ls -ll / + - ls -ll /bzl_cache/ + - touch /bzl_cache/toto + - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME || + - docker pull $CI_REGISTRY_IMAGE:cpu-basic-test || + - > + docker build + --pull + --network="host" + --cache-from $CI_REGISTRY_IMAGE:cpu-basic-test + --cache-from $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME + --label "org.opencontainers.image.title=$CI_PROJECT_TITLE" + --label "org.opencontainers.image.url=$CI_PROJECT_URL" + --label "org.opencontainers.image.created=$CI_JOB_STARTED_AT" + --label "org.opencontainers.image.revision=$CI_COMMIT_SHA" + --label "org.opencontainers.image.version=$CI_COMMIT_REF_NAME" + --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME + --build-arg OTBTESTS="true" + --build-arg KEEP_SRC_OTB="true" + --build-arg BZL_CONFIGS="" + --build-arg BASE_IMG="ubuntu:20.04" + . + - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME -flake8: +.static_analysis_base: stage: Static Analysis allow_failure: true + +flake8: + extends: .static_analysis_base script: - sudo apt update && sudo apt install flake8 -y - python -m flake8 --max-line-length=120 $OTBTF_SRC/python pylint: - stage: Static Analysis - allow_failure: true + extends: .static_analysis_base script: - sudo apt update && sudo apt install pylint -y - pylint --disable=too-many-nested-blocks,too-many-locals,too-many-statements,too-few-public-methods,too-many-instance-attributes,too-many-arguments --ignored-modules=tensorflow --max-line-length=120 --logging-format-style=new $OTBTF_SRC/python codespell: - stage: Static Analysis - allow_failure: true + extends: .static_analysis_base script: - sudo pip install codespell && codespell cppcheck: - stage: Static Analysis - allow_failure: true + extends: .static_analysis_base script: - sudo apt update && sudo apt install cppcheck -y - cd $OTBTF_SRC/ && cppcheck --enable=all --error-exitcode=1 -I include/ --suppress=missingInclude --suppress=unusedFunction . +.tests_base: + artifacts: + paths: + - $ARTIFACT_TEST_DIR/*.* + expire_in: 1 week + when: on_failure + ctest: + extends: .tests_base stage: Test script: - - *compile_otbtf - - sudo rm -rf $OTB_TEST_DIR/* # Empty testing temporary folder (old files here) - cd $OTB_BUILD/ && sudo ctest -L OTBTensorflow # Run ctest after_script: - cp -r $OTB_TEST_DIR $ARTIFACT_TEST_DIR - artifacts: - paths: - - $ARTIFACT_TEST_DIR/*.* - expire_in: 1 week - when: on_failure .applications_test_base: + extends: .tests_base stage: Applications Test rules: # Only for MR targeting 'develop' and 'master' branches because applications tests are slow - if: $CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'develop' - if: $CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'master' - artifacts: - when: on_failure - paths: - - $CI_PROJECT_DIR/report_*.xml - - $ARTIFACT_TEST_DIR/*.* - expire_in: 1 week - + before_script: + - pip3 install pytest pytest-cov pytest-order + - mkdir -p $ARTIFACT_TEST_DIR + - cd $CI_PROJECT_DIR + crc_book: extends: .applications_test_base script: - - *compile_otbtf - - *install_pytest - - cd $CI_PROJECT_DIR - mkdir -p $CRC_BOOK_TMP - TMPDIR=$CRC_BOOK_TMP DATADIR=$CI_PROJECT_DIR/test/data python -m pytest --junitxml=$CI_PROJECT_DIR/report_tutorial.xml $OTBTF_SRC/test/tutorial_unittest.py after_script: - - mkdir -p $ARTIFACT_TEST_DIR - cp $CRC_BOOK_TMP/*.* $ARTIFACT_TEST_DIR/ sr4rs: extends: .applications_test_base script: - - *compile_otbtf - - *install_pytest - - cd $CI_PROJECT_DIR - wget -O sr4rs_sentinel2_bands4328_france2020_savedmodel.zip https://nextcloud.inrae.fr/s/EZL2JN7SZyDK8Cf/download/sr4rs_sentinel2_bands4328_france2020_savedmodel.zip - unzip -o sr4rs_sentinel2_bands4328_france2020_savedmodel.zip @@ -116,5 +128,13 @@ sr4rs: - rm -rf sr4rs - git clone https://github.com/remicres/sr4rs.git - export PYTHONPATH=$PYTHONPATH:$PWD/sr4rs - - python -m pytest --junitxml=$CI_PROJECT_DIR/report_sr4rs.xml $OTBTF_SRC/test/sr4rs_unittest.py + - python -m pytest --junitxml=$ARTIFACT_TEST_DIR/report_sr4rs.xml $OTBTF_SRC/test/sr4rs_unittest.py +deploy: + stage: Ship + only: + - master + script: + - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME + - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME $CI_REGISTRY_IMAGE:cpu-basic-test + - docker push $CI_REGISTRY_IMAGE:cpu-basic-test diff --git a/Dockerfile b/Dockerfile index 28ee7875d9c80324d48f3b476371d89c1b53bb25..4a4d3ab7cea36c05d740c0e15d75ad16bc76ba85 100644 --- a/Dockerfile +++ b/Dockerfile @@ -85,6 +85,7 @@ RUN git clone --single-branch -b $TF https://github.com/tensorflow/tensorflow.gi ### OTB ARG GUI=false ARG OTB=7.4.0 +ARG OTBTESTS=false RUN mkdir /src/otb WORKDIR /src/otb @@ -97,6 +98,8 @@ RUN apt-get update -y \ && git clone --single-branch -b $OTB https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb.git \ && mkdir -p build \ && cd build \ + && if $OTBTESTS; then \ + echo "-DBUILD_TESTING=ON" >> ../build-flags-otb.txt; fi \ # Set GL/Qt build flags && if $GUI; then \ sed -i -r "s/-DOTB_USE_(QT|OPENGL|GL[UFE][WT])=OFF/-DOTB_USE_\1=ON/" ../build-flags-otb.txt; fi \ diff --git a/README.md b/README.md index d1993d5bb839fea504524e9cd21c4d120bf07335..343c4863ccad0ea295b9a29b243e3cc518e9c0e0 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ # OTBTF: Orfeo ToolBox meets TensorFlow [](https://opensource.org/licenses/Apache-2.0) +[](https://gitlab.irstea.fr/remi.cresson/otbtf/-/commits/develop) This remote module of the [Orfeo ToolBox](https://www.orfeo-toolbox.org) provides a generic, multi purpose deep learning framework, targeting remote sensing images processing. It contains a set of new process objects that internally invoke [Tensorflow](https://www.tensorflow.org/), and a bunch of user-oriented applications to perform deep learning with real-world remote sensing images.