Unable to download HLS data: 403 error but authenticated
Posted: Wed Aug 28, 2024 4:31 pm America/New_York
I'm using the earthaccess Python library (version 0.10.0) that's available on PyPi to download HLS data.
I'm able to search for items but when I try to download an asset (i.e., a tif that pertains to a band), I get a 403 error.
My code is below.
import earthaccess
import geopandas as gpd
from osgeo import gdal
# Ordering: blue, green, red, nir08, swir16, swir22, ndvi, high, low, vege, soil, qa
COLLECTIONS = ["HLSL30", "HLSS30"]
COMMON_BANDS = ["blue", "green", "red", "nir08", "swir16", "swir22", "qa"]
COMMON_NDVI_BANDS = ["blue", "green", "red", "nir08", "swir16", "swir22", "ndvi", "qa"]
L8_BANDS = ["B02", "B03", "B04", "B05", "B06", "B07", "Fmask"]
S2_BANDS = ["B02", "B03", "B04", "B8A", "B11", "B12", "Fmask"]
NODATA_VALUE = -9999
START_DATE = "2014-01-01"
END_DATE = "2021-12-31"
MAX_RETRIES = 5
def make_item_dict(items):
from collections import defaultdict
item_di = defaultdict(lambda: [])
for item in items:
date = dayofyear_to_date(item["meta"]["native-id"])
item_di[date].append(item)
return item_di
def parse_asset_link(item_link: str) -> Tuple[str, str, str, str]:
"""
Parse earthaccess asset link to extract sensor and band name.
Arguments:
item_link: earthaccess item link.
Returns:
Sensor, common band name, canonical band name, & item link.
Used by: parse_item_links.
"""
parts = Path(item_link).stem.split(".")
sensor = parts[1]
canonical = parts[-1]
if (sensor == "S30") and (canonical in S2_BANDS):
common_ix = S2_BANDS.index(canonical)
elif (sensor == "L30") and (canonical in L8_BANDS):
common_ix = L8_BANDS.index(canonical)
else:
return None
common = COMMON_BANDS[common_ix]
return (sensor, common, canonical, item_link)
gdal.SetConfigOption("GDAL_HTTP_COOKIEFILE", str(args.cookies_src))
gdal.SetConfigOption("GDAL_HTTP_COOKIEJAR", str(args.cookies_src))
gdal.SetConfigOption("GDAL_DISABLE_READDIR_ON_OPEN", "EMPTY_DIR")
gdal.SetConfigOption("CPL_VSIL_CURL_ALLOWED_EXTENSIONS", "TIF")
gdal.SetConfigOption("GDAL_HTTP_UNSAFESSL", "YES")
chunksize = dict(band=1, x=args.chunksize, y=args.chunksize)
earthaccess.login(strategy="netrc", persist=True)
logger.info("Load region model and query earthaccess.")
output_bands = LSMA_BANDS if run_lsma else COMMON_NDVI_BANDS
# Example geojson link here: https://github.com/pubgeo/IARPA-SMART/blob/add_annotations/annotations/empty_region_models/BR_R002.geojson
region_model = gpd.read_file(args.region_model_src) # crs is already 4326
bbox = xmin, ymin, xmax, ymax = tuple(list(region_model.total_bounds))
polygon = Polygon.from_bounds(xmin, ymin, xmax, ymax)
bounds_gdf = gpd.GeoDataFrame(dict(geometry=[polygon]), crs=4326)
temporal = (f"{START_DATE}T00:00:00", f"{END_DATE}T23:59:59")
items = earthaccess.search_data(
short_name=COLLECTIONS,
bounding_box=bbox,
temporal=temporal,
count=args.results_count,
)
item_dict = make_item_dict(items)
for date, item_li in item_dict.items():
# we can have multiple items for one date depending on bounding box
for item in item_li:
asset_links = item.data_links()
for asset_link in asset_links:
parsed = parse_asset_link(asset_link)
_, common, canonical, href = parsed
for i in range(MAX_RETRIES):
xarr = rxr.open_rasterio(href, chunks=chunksize, masked=True).squeeze("band", drop=True)
# ^^this is where I get the 403 error
I'm able to search for items but when I try to download an asset (i.e., a tif that pertains to a band), I get a 403 error.
My code is below.
import earthaccess
import geopandas as gpd
from osgeo import gdal
# Ordering: blue, green, red, nir08, swir16, swir22, ndvi, high, low, vege, soil, qa
COLLECTIONS = ["HLSL30", "HLSS30"]
COMMON_BANDS = ["blue", "green", "red", "nir08", "swir16", "swir22", "qa"]
COMMON_NDVI_BANDS = ["blue", "green", "red", "nir08", "swir16", "swir22", "ndvi", "qa"]
L8_BANDS = ["B02", "B03", "B04", "B05", "B06", "B07", "Fmask"]
S2_BANDS = ["B02", "B03", "B04", "B8A", "B11", "B12", "Fmask"]
NODATA_VALUE = -9999
START_DATE = "2014-01-01"
END_DATE = "2021-12-31"
MAX_RETRIES = 5
def make_item_dict(items):
from collections import defaultdict
item_di = defaultdict(lambda: [])
for item in items:
date = dayofyear_to_date(item["meta"]["native-id"])
item_di[date].append(item)
return item_di
def parse_asset_link(item_link: str) -> Tuple[str, str, str, str]:
"""
Parse earthaccess asset link to extract sensor and band name.
Arguments:
item_link: earthaccess item link.
Returns:
Sensor, common band name, canonical band name, & item link.
Used by: parse_item_links.
"""
parts = Path(item_link).stem.split(".")
sensor = parts[1]
canonical = parts[-1]
if (sensor == "S30") and (canonical in S2_BANDS):
common_ix = S2_BANDS.index(canonical)
elif (sensor == "L30") and (canonical in L8_BANDS):
common_ix = L8_BANDS.index(canonical)
else:
return None
common = COMMON_BANDS[common_ix]
return (sensor, common, canonical, item_link)
gdal.SetConfigOption("GDAL_HTTP_COOKIEFILE", str(args.cookies_src))
gdal.SetConfigOption("GDAL_HTTP_COOKIEJAR", str(args.cookies_src))
gdal.SetConfigOption("GDAL_DISABLE_READDIR_ON_OPEN", "EMPTY_DIR")
gdal.SetConfigOption("CPL_VSIL_CURL_ALLOWED_EXTENSIONS", "TIF")
gdal.SetConfigOption("GDAL_HTTP_UNSAFESSL", "YES")
chunksize = dict(band=1, x=args.chunksize, y=args.chunksize)
earthaccess.login(strategy="netrc", persist=True)
logger.info("Load region model and query earthaccess.")
output_bands = LSMA_BANDS if run_lsma else COMMON_NDVI_BANDS
# Example geojson link here: https://github.com/pubgeo/IARPA-SMART/blob/add_annotations/annotations/empty_region_models/BR_R002.geojson
region_model = gpd.read_file(args.region_model_src) # crs is already 4326
bbox = xmin, ymin, xmax, ymax = tuple(list(region_model.total_bounds))
polygon = Polygon.from_bounds(xmin, ymin, xmax, ymax)
bounds_gdf = gpd.GeoDataFrame(dict(geometry=[polygon]), crs=4326)
temporal = (f"{START_DATE}T00:00:00", f"{END_DATE}T23:59:59")
items = earthaccess.search_data(
short_name=COLLECTIONS,
bounding_box=bbox,
temporal=temporal,
count=args.results_count,
)
item_dict = make_item_dict(items)
for date, item_li in item_dict.items():
# we can have multiple items for one date depending on bounding box
for item in item_li:
asset_links = item.data_links()
for asset_link in asset_links:
parsed = parse_asset_link(asset_link)
_, common, canonical, href = parsed
for i in range(MAX_RETRIES):
xarr = rxr.open_rasterio(href, chunks=chunksize, masked=True).squeeze("band", drop=True)
# ^^this is where I get the 403 error