Browse Source

sat polish and enhance, cf #4

master
Olivier Courtin 1 month ago
parent
commit
b154fb7f5e
2 changed files with 48 additions and 30 deletions
  1. +5
    -6
      docs/tools.md
  2. +43
    -24
      neat_eo/tools/sat.py

+ 5
- 6
docs/tools.md View File

@@ -262,9 +262,9 @@ Web UI:
```nosyn
usage: neo sat [-h] [--config CONFIG] [--pg PG] [--cover COVER]
[--granules GRANULES [GRANULES ...]] [--scenes SCENES]
[--level {2A,3A}] [--start START] [--end END] [--clouds CLOUDS]
[--limit LIMIT] [--download] [--workers WORKERS]
[--timeout TIMEOUT] [--out [OUT]]
[--start START] [--end END] [--clouds CLOUDS] [--limit LIMIT]
[--download] [--workers WORKERS] [--timeout TIMEOUT]
[--out [OUT]]

optional arguments:
-h, --help show this help message and exit
@@ -278,7 +278,6 @@ Spatial extent [one among the following is required]:
--scenes SCENES Path to a Scenes UUID file

Filters:
--level {2A,3A} Processing Level
--start START YYYY-MM-DD starting date
--end END YYYY-MM-DD end date
--clouds CLOUDS max threshold for cloud coverage [0-100]
@@ -286,8 +285,8 @@ Filters:

Download:
--download if set, perform also download operation.
--workers WORKERS number of workers [default: 4]
--timeout TIMEOUT download request timeout (in seconds) [default: 180]
--workers WORKERS number of workers [default: 10]
--timeout TIMEOUT request timeout in seconds [default: 180]
```
## neo subset
```nosyn


+ 43
- 24
neat_eo/tools/sat.py View File

@@ -4,6 +4,7 @@ import sys
import json
import glob

import urllib3
import hashlib
import requests
import concurrent.futures as futures
@@ -27,7 +28,6 @@ def add_parser(subparser, formatter_class):
extent.add_argument("--scenes", type=str, help="Path to a Scenes UUID file")

filters = parser.add_argument_group("Filters")
filters.add_argument("--level", type=str, choices=["2A", "3A"], help="Processing Level")
filters.add_argument("--start", type=str, help="YYYY-MM-DD starting date")
filters.add_argument("--end", type=str, help="YYYY-MM-DD end date")
filters.add_argument("--clouds", type=int, help="max threshold for cloud coverage [0-100]")
@@ -35,8 +35,8 @@ def add_parser(subparser, formatter_class):

dl = parser.add_argument_group("Download")
dl.add_argument("--download", action="store_true", help="if set, perform also download operation.")
dl.add_argument("--workers", type=int, default=4, help="number of workers [default: 4]")
dl.add_argument("--timeout", type=int, default=180, help="download request timeout (in seconds) [default: 180]")
dl.add_argument("--workers", type=int, default=10, help="number of workers [default: 10]")
dl.add_argument("--timeout", type=int, default=180, help="request timeout in seconds [default: 180]")

parser.add_argument("--out", type=str, nargs="?", help="output directory path [required if download is set]")
parser.set_defaults(func=main)
@@ -70,14 +70,13 @@ def md5(path):
def search_scenes(args, log):

scenes = []
log.log("----------------------------------------------------------------------------------------------")
log.log(" Granule Date Clouds Scene UUID Processing ")
log.log("----------------------------------------------------------------------------------------------")
log.log("--------------------------------------------------------------------------")
log.log(" Granule Date Clouds Scene UUID ")
log.log("--------------------------------------------------------------------------")

for granule in args.granules:
data = {"location": "T" + granule, "maxRecords": 500}
if args.level:
data["processingLevel"] = "LEVEL" + args.level
data["processingLevel"] = "LEVEL2A" # hardcoded
if args.start:
data["startDate"] = args.start
if args.end:
@@ -88,7 +87,7 @@ def search_scenes(args, log):
assert data, "Unable to perform query: {}".format(url)

if not len(data["features"]):
log.log(" {} NOTICE: No scene found".format(granule))
log.log(" {} NOTICE: No scene found".format(granule))
continue

features = (
@@ -108,11 +107,7 @@ def search_scenes(args, log):
"dir": feature["properties"]["title"],
}
)
log.log(
" {} {} {} {} {}".format(
granule, date, cover, feature["id"], feature["properties"]["processingLevel"]
)
)
log.log(" {} {} {} {}".format(granule, date, cover, feature["id"]))

return scenes

@@ -156,22 +151,46 @@ def main(args):
scene["dir"] = glob.glob(scene_dir + "*")[0]
return scene, None, True # Already Downloaded

token = get_token(login, password)
url = THEIA_URL + "/resto2/collections/SENTINEL2/{}/download/?issuerId=theia".format(scene["uuid"])
resp = requests.get(url, headers={"Authorization": "Bearer {}".format(token)}, stream=True)
if resp is None:
return scene, None, False # Auth issue

zip_path = os.path.join(args.out, scene["uuid"] + ".zip")
with open(zip_path, "wb") as fp:
progress = tqdm(unit="B", unit_scale=True, desc=scene["uuid"], total=int(resp.headers["Content-Length"]))

try:
fp = open(zip_path, "wb")
assert fp, "Unable to write in {}".format(zip_path)

token = get_token(login, password)
resp = requests.get(
url, headers={"Authorization": "Bearer {}".format(token)}, stream=True, timeout=args.timeout
)
length = int(resp.headers["Content-Length"])
progress = tqdm(unit="B", unit_scale=True, ascii=True, desc=scene["uuid"], total=length)
for chunk in resp.iter_content(chunk_size=16384):
progress.update(16384)
fp.write(chunk)
progress.update(16384)

return scene, zip_path, True
except ( # Auth token time validity handling
urllib3.exceptions.ProtocolError,
ConnectionResetError,
ConnectionError,
requests.exceptions.ConnectionError,
requests.exceptions.ChunkedEncodingError,
):
fp = open(zip_path, "wb")
assert fp, "Unable to write in {}".format(zip_path)

token = get_token(login, password)
resp = requests.get(
url, headers={"Authorization": "Bearer {}".format(token)}, stream=True, timeout=args.timeout
)
if progress:
progress.reset()
for chunk in resp.iter_content(chunk_size=16384):
fp.write(chunk)
progress.update(16384)

return scene, None, False # Write issue
if resp is None:
return scene, None, False
return scene, zip_path, True

for scene, zip_path, ok in executor.map(worker, scenes):
if zip_path and md5(zip_path) == scene["checksum"]:


Loading…
Cancel
Save