Efficient AI4EO OpenSource framework
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

169 lines
10KB

  1. help:
  2. @echo "This Makefile rules are designed for Neat-EO.pink devs and power-users."
  3. @echo "For plain user installation follow README.md instructions, instead."
  4. @echo ""
  5. @echo ""
  6. @echo " make install To install, few Python dev tools and Neat-EO.pink in editable mode."
  7. @echo " So any further Neat-EO.pink Python code modification will be usable at once,"
  8. @echo " throught either neo tools commands or neat_eo.* modules."
  9. @echo ""
  10. @echo " make check Launchs code tests, and tools doc updating."
  11. @echo " Do it, at least, before sending a Pull Request."
  12. @echo ""
  13. @echo " make check_tuto Launchs neo commands embeded in tutorials, to be sure everything still up to date."
  14. @echo " Do it, at least, on each CLI modifications, and before a release."
  15. @echo " NOTA: It takes a while."
  16. @echo ""
  17. @echo " make pink Python code beautifier,"
  18. @echo " as Pink is the new Black ^^"
  19. # Dev install
  20. install:
  21. sudo apt install gdal-bin osmium-tool
  22. pip3 install pytest black flake8 twine
  23. pip3 install -e .
  24. # Lauch all tests
  25. check: ut it doc
  26. @echo "==================================================================================="
  27. @echo "All tests passed !"
  28. @echo "==================================================================================="
  29. # Python code beautifier
  30. pink:
  31. black -l 125 *.py neat_eo/*.py neat_eo/*/*.py tests/*py tests/*/*.py
  32. # Perform units tests, and linter checks
  33. ut:
  34. @echo "==================================================================================="
  35. black -l 125 --check *.py neat_eo/*.py neat_eo/*/*.py
  36. @echo "==================================================================================="
  37. flake8 --max-line-length 125 --ignore=E203,E241,E226,E272,E261,E221,W503,E722
  38. @echo "==================================================================================="
  39. pytest tests -W ignore::UserWarning
  40. # Launch Integration Tests
  41. it: it_pre it_train it_post
  42. # Integration Tests: Data Preparation
  43. it_pre:
  44. @echo "==================================================================================="
  45. rm -rf it
  46. neo info
  47. neo cover --zoom 18 --bbox 4.8,45.7,4.82,45.72 --out it/cover
  48. neo download --rate 20 --type WMS --url "https://download.data.grandlyon.com/wms/grandlyon?SERVICE=WMS&REQUEST=GetMap&VERSION=1.3.0&LAYERS=Ortho2015_vue_ensemble_16cm_CC46&WIDTH=512&HEIGHT=512&CRS=EPSG:3857&BBOX={xmin},{ymin},{xmax},{ymax}&FORMAT=image/jpeg" --cover it/cover --out it/images
  49. echo "Download Buildings GeoJSON" && wget --show-progress -q -nc -O it/lyon_roofprint.json "https://download.data.grandlyon.com/wfs/grandlyon?SERVICE=WFS&REQUEST=GetFeature&TYPENAME=ms:fpc_fond_plan_communaut.fpctoit&VERSION=1.1.0&srsName=EPSG:4326&BBOX=4.79,45.69,4.83,45.73&outputFormat=application/json; subtype=geojson" | true
  50. echo "Download Roads GeoJSON" && wget --show-progress -q -nc -O it/lyon_road.json "https://download.data.grandlyon.com/wfs/grandlyon?SERVICE=WFS&VERSION=1.1.0&request=GetFeature&typename=pvo_patrimoine_voirie.pvochausseetrottoir&outputFormat=application/json; subtype=geojson&SRSNAME=EPSG:4326&bbox=`neo cover --dir it/images --type extent`" | true
  51. ogr2ogr -f SQLite it/lyon_road.sqlite it/lyon_road.json -dsco SPATIALITE=YES -t_srs EPSG:3857 -nln roads -lco GEOMETRY_NAME=geom
  52. ogr2ogr -f GeoJSON it/lyon_road_poly.json it/lyon_road.sqlite -dialect sqlite -sql "SELECT Buffer(geom, IFNULL(largeurchaussee, 5.0) / 2.0) AS geom FROM roads"
  53. neo rasterize --type Building --geojson it/lyon_roofprint.json --config config.toml --cover it/cover --out it/labels
  54. neo rasterize --type Road --geojson it/lyon_road_poly.json --config config.toml --cover it/cover --append --out it/labels
  55. neo rasterize --type Building --geojson it/lyon_roofprint.json --config config.toml --cover it/cover --out it/labels_osm
  56. neo cover --dir it/images --splits 80/20 --out it/train/cover it/eval/cover
  57. neo subset --dir it/images --cover it/train/cover --out it/train/images
  58. neo subset --dir it/labels --cover it/train/cover --out it/train/labels
  59. neo subset --dir it/images --cover it/eval/cover --out it/eval/images
  60. neo subset --dir it/labels --cover it/eval/cover --out it/eval/labels
  61. mkdir --parents it/predict/tiff
  62. wget -nc -O it/predict/tiff/1841_5174_08_CC46.tif "https://download.data.grandlyon.com/files/grandlyon/imagerie/ortho2018/ortho/GeoTiff_YcBcR/1km_8cm_CC46/1841_5174_08_CC46.tif"
  63. wget -nc -O it/predict/tiff/1842_5174_08_CC46.tif "https://download.data.grandlyon.com/files/grandlyon/imagerie/ortho2018/ortho/GeoTiff_YcBcR/1km_8cm_CC46/1842_5174_08_CC46.tif"
  64. neo tile --zoom 18 --rasters it/predict/tiff/*.tif --out it/predict/images
  65. neo cover --zoom 18 --dir it/predict/images --out it/predict/cover
  66. echo "Download PBF" && wget -nc -O it/predict/ra.pbf "http://download.geofabrik.de/europe/france/rhone-alpes-latest.osm.pbf" | true
  67. osmium extract --bbox `neo cover --dir it/predict/images --type extent` -o it/predict/lyon.pbf it/predict/ra.pbf
  68. neo extract --type Building --pbf it/predict/lyon.pbf --out it/predict/osm_building.json
  69. neo extract --type Road --pbf it/predict/lyon.pbf --out it/predict/osm_road.json
  70. neo rasterize --type Building --geojson it/predict/osm_building.json --config config.toml --cover it/predict/cover --out it/predict/labels
  71. neo rasterize --type Road --geojson it/predict/osm_road.json --config config.toml --cover it/predict/cover --append --out it/predict/labels
  72. # Integration Tests: Training
  73. it_train:
  74. @echo "==================================================================================="
  75. export CUDA_VISIBLE_DEVICES=0 && neo train --config config.toml --bs 4 --lr 0.00025 --epochs 2 --dataset it/train --classes_weights `neo dataset --mode weights --dataset it/train --config config.toml` --out it/pth
  76. export CUDA_VISIBLE_DEVICES=0,1 && neo train --config config.toml --bs 4 --lr 0.00025 --epochs 4 --resume --checkpoint it/pth/checkpoint-00002.pth --classes_weights auto --dataset it/train --out it/pth
  77. export CUDA_VISIBLE_DEVICES=0,1 && neo train --config config.toml --bs 4 --optimizer AdamW --lr 0.00025 --epochs 6 --resume --checkpoint it/pth/checkpoint-00004.pth --classes_weights auto --dataset it/train --out it/pth
  78. neo eval --config config.toml --bs 4 --checkpoint it/pth/checkpoint-00006.pth --classes_weights auto --dataset it/eval
  79. neo info --checkpoint it/pth/checkpoint-00006.pth
  80. # Integration Tests: Post Training
  81. it_post:
  82. @echo "==================================================================================="
  83. neo export --checkpoint it/pth/checkpoint-00006.pth --type jit --out it/pth/export.jit
  84. neo export --checkpoint it/pth/checkpoint-00006.pth --type onnx --out it/pth/export.onnx
  85. neo predict --config config.toml --checkpoint it/pth/checkpoint-00006.pth --dataset it/predict --out it/predict/masks
  86. neo predict --metatiles --config config.toml --checkpoint it/pth/checkpoint-00006.pth --dataset it/predict --out it/predict/masks_meta
  87. neo predict --metatiles --keep_borders --config config.toml --checkpoint it/pth/checkpoint-00006.pth --dataset it/predict --out it/predict/masks_keep
  88. neo cover --dir it/predict/masks_meta --out it/predict/cover
  89. neo compare --cover it/predict/cover --config config.toml --images it/predict/images it/predict/labels it/predict/masks --mode stack --labels it/predict/labels --masks it/predict/masks_meta --out it/predict/compare
  90. neo compare --cover it/predict/cover --images it/predict/images it/predict/compare --mode side --out it/predict/compare_side
  91. neo compare --cover it/predict/cover --config config.toml --mode list --labels it/predict/labels --max Building QoD 0.50 --masks it/predict/masks_meta --geojson --out it/predict/compare/tiles.json
  92. cp it/predict/compare/tiles.json it/predict/compare_side/tiles.json
  93. neo vectorize --type Building --config config.toml --masks it/predict/masks_meta --out it/predict/building.json
  94. neo vectorize --type Road --config config.toml --masks it/predict/masks_meta --out it/predict/road.json
  95. # Documentation generation (tools and config file)
  96. doc:
  97. @echo "==================================================================================="
  98. @echo "# Neat-EO.pink tools documentation" > docs/tools.md
  99. @for tool in `ls neat_eo/tools/[^_]*py | sed -e 's#.*/##g' -e 's#.py##'`; do \
  100. echo "Doc generation: $$tool"; \
  101. echo "## neo $$tool" >> docs/tools.md; \
  102. echo '```nosyn' >> docs/tools.md; \
  103. neo $$tool -h >> docs/tools.md; \
  104. echo '```' >> docs/tools.md; \
  105. done
  106. @echo "Doc generation: config.toml"
  107. @echo "## config.toml" > docs/config.md; \
  108. echo '```nosyn' >> docs/config.md; \
  109. cat config.toml >> docs/config.md; \
  110. echo '```' >> docs/config.md;
  111. @echo "Doc generation: Makefile"
  112. @echo "## Makefile" > docs/makefile.md; \
  113. echo '```nosyn' >> docs/makefile.md; \
  114. make --no-print-directory >> docs/makefile.md; \
  115. echo '```' >> docs/makefile.md;
  116. # Check neo commands embeded in Documentation
  117. check_doc:
  118. @echo "==================================================================================="
  119. @echo "Checking README:"
  120. @echo "==================================================================================="
  121. @rm -rf ds && sed -n -e '/```bash/,/```/ p' README.md | sed -e '/```/d' > .CHECK && sh .CHECK
  122. @echo "==================================================================================="
  123. # Check neo commands embeded in Tutorials
  124. check_tuto: check_101 check_qa
  125. check_101:
  126. @echo "==================================================================================="
  127. @echo "Checking 101"
  128. @mkdir -p tuto && cd tuto && mkdir 101 && sed -n -e '/```bash/,/```/ p' ../docs/101.md | sed -e '/```/d' > 101/.CHECK && cd 101 && sh .CHECK && cd ../..
  129. @cd tuto/101 && tar cf 101.tar train/images train/labels predict/masks predict/compare predict/compare_side && cd ..
  130. check_qa:
  131. @echo "==================================================================================="
  132. @echo "Checking QA"
  133. @mkdir -p tuto && cd tuto && mkdir qa && sed -n -e '/```bash/,/```/ p' ../docs/qa.md | sed -e '/```/d' > qa/.CHECK && cd qa && sh .CHECK && cd ../..
  134. # Send a release on PyPI
  135. pypi:
  136. rm -rf dist Neat-EO.pink.egg-info
  137. python3 setup.py sdist
  138. twine upload dist/* -r pypi