Earthdata Forum

I am trying to download 240 scenes from the asf vertex. Please help me. What is the best way to download a large number of scenes? Am I doing it inefficiently? some of the urls for the secenes are:
https://datapool.asf.alaska.edu/SLC/SA/S1A_IW_SLC__1SDV_20241116T062301_20241116T062329_056576_06F01D_BACB.zip
https://datapool.asf.alaska.edu/SLC/SA/S1A_IW_SLC__1SDV_20241114T175034_20241114T175101_056554_06EF3C_5AF9.zip
https://datapool.asf.alaska.edu/SLC/SA/S1A_IW_SLC__1SDV_20241111T061442_20241111T061509_056503_06ED34_C50F.zip
https://datapool.asf.alaska.edu/SLC/SA/S1A_IW_SLC__1SDV_20241107T175845_20241107T175912_056452_06EB2B_7A69.zip
https://datapool.asf.alaska.edu/SLC/SA/S1A_IW_SLC__1SDV_20241104T062301_20241104T062329_056401_06E920_DC3E.zip

This is my code:
session = asf.ASFSession().auth_with_creds(' ',' ')
urls = []
with open('URLs.txt', 'r') as file:
for line in file:
urls.append(line.strip())
asf.download_urls(urls=urls, path='D:/spenfarm_scenes_2021_2023/spenfarm_data', session=session)

This is the error i am getting:
{
"name": "ConnectionError",
"message": "('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))",
"stack": "---------------------------------------------------------------------------
RemoteDisconnected Traceback (most recent call last)
File c:\\Users\\ttgvytft90\\AppData\\Local\\anaconda3\\Lib\\site-packages\\urllib3\\connectionpool.py:789, in HTTPConnectionPool.urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)
788 # Make the request on the HTTPConnection object
--> 789 response = self._make_request(
790 conn,
791 method,
792 url,
793 timeout=timeout_obj,
794 body=body,
795 headers=headers,
796 chunked=chunked,
797 retries=retries,
798 response_conn=response_conn,
799 preload_content=preload_content,
800 decode_content=decode_content,
801 **response_kw,
802 )
804 # Everything went great!

File c:\\Users\\ttgvytft90\\AppData\\Local\\anaconda3\\Lib\\site-packages\\urllib3\\connectionpool.py:536, in HTTPConnectionPool._make_request(self, conn, method, url, body, headers, retries, timeout, chunked, response_conn, preload_content, decode_content, enforce_content_length)
535 try:
--> 536 response = conn.getresponse()
537 except (BaseSSLError, OSError) as e:

File c:\\Users\\ttgvytft90\\AppData\\Local\\anaconda3\\Lib\\site-packages\\urllib3\\connection.py:507, in HTTPConnection.getresponse(self)
506 # Get the response from http.client.HTTPConnection
--> 507 httplib_response = super().getresponse()
509 try:

File c:\\Users\\ttgvytft90\\AppData\\Local\\anaconda3\\Lib\\http\\client.py:1428, in HTTPConnection.getresponse(self)
1427 try:
-> 1428 response.begin()
1429 except ConnectionError:

File c:\\Users\\ttgvytft90\\AppData\\Local\\anaconda3\\Lib\\http\\client.py:331, in HTTPResponse.begin(self)
330 while True:
--> 331 version, status, reason = self._read_status()
332 if status != CONTINUE:

File c:\\Users\\ttgvytft90\\AppData\\Local\\anaconda3\\Lib\\http\\client.py:300, in HTTPResponse._read_status(self)
297 if not line:
298 # Presumably, the server closed the connection before
299 # sending a valid response.
--> 300 raise RemoteDisconnected(\"Remote end closed connection without\"
301 \" response\")
302 try:

RemoteDisconnected: Remote end closed connection without response

During handling of the above exception, another exception occurred:

ProtocolError Traceback (most recent call last)
File c:\\Users\\ttgvytft90\\AppData\\Local\\anaconda3\\Lib\\site-packages\\requests\\adapters.py:667, in HTTPAdapter.send(self, request, stream, timeout, verify, cert, proxies)
666 try:
--> 667 resp = conn.urlopen(
668 method=request.method,
669 url=url,
670 body=request.body,
671 headers=request.headers,
672 redirect=False,
673 assert_same_host=False,
674 preload_content=False,
675 decode_content=False,
676 retries=self.max_retries,
677 timeout=timeout,
678 chunked=chunked,
679 )
681 except (ProtocolError, OSError) as err:

File c:\\Users\\ttgvytft90\\AppData\\Local\\anaconda3\\Lib\\site-packages\\urllib3\\connectionpool.py:843, in HTTPConnectionPool.urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)
841 new_e = ProtocolError(\"Connection aborted.\", new_e)
--> 843 retries = retries.increment(
844 method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]
845 )
846 retries.sleep()

File c:\\Users\\ttgvytft90\\AppData\\Local\\anaconda3\\Lib\\site-packages\\urllib3\\util\\retry.py:474, in Retry.increment(self, method, url, response, error, _pool, _stacktrace)
473 if read is False or method is None or not self._is_method_retryable(method):
--> 474 raise reraise(type(error), error, _stacktrace)
475 elif read is not None:

File c:\\Users\\ttgvytft90\\AppData\\Local\\anaconda3\\Lib\\site-packages\\urllib3\\util\\util.py:38, in reraise(tp, value, tb)
37 if value.__traceback__ is not tb:
---> 38 raise value.with_traceback(tb)
39 raise value

File c:\\Users\\ttgvytft90\\AppData\\Local\\anaconda3\\Lib\\site-packages\\urllib3\\connectionpool.py:789, in HTTPConnectionPool.urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)
788 # Make the request on the HTTPConnection object
--> 789 response = self._make_request(
790 conn,
791 method,
792 url,
793 timeout=timeout_obj,
794 body=body,
795 headers=headers,
796 chunked=chunked,
797 retries=retries,
798 response_conn=response_conn,
799 preload_content=preload_content,
800 decode_content=decode_content,
801 **response_kw,
802 )
804 # Everything went great!

File c:\\Users\\ttgvytft90\\AppData\\Local\\anaconda3\\Lib\\site-packages\\urllib3\\connectionpool.py:536, in HTTPConnectionPool._make_request(self, conn, method, url, body, headers, retries, timeout, chunked, response_conn, preload_content, decode_content, enforce_content_length)
535 try:
--> 536 response = conn.getresponse()
537 except (BaseSSLError, OSError) as e:

File c:\\Users\\ttgvytft90\\AppData\\Local\\anaconda3\\Lib\\site-packages\\urllib3\\connection.py:507, in HTTPConnection.getresponse(self)
506 # Get the response from http.client.HTTPConnection
--> 507 httplib_response = super().getresponse()
509 try:

File c:\\Users\\ttgvytft90\\AppData\\Local\\anaconda3\\Lib\\http\\client.py:1428, in HTTPConnection.getresponse(self)
1427 try:
-> 1428 response.begin()
1429 except ConnectionError:

File c:\\Users\\ttgvytft90\\AppData\\Local\\anaconda3\\Lib\\http\\client.py:331, in HTTPResponse.begin(self)
330 while True:
--> 331 version, status, reason = self._read_status()
332 if status != CONTINUE:

File c:\\Users\\ttgvytft90\\AppData\\Local\\anaconda3\\Lib\\http\\client.py:300, in HTTPResponse._read_status(self)
297 if not line:
298 # Presumably, the server closed the connection before
299 # sending a valid response.
--> 300 raise RemoteDisconnected(\"Remote end closed connection without\"
301 \" response\")
302 try:

ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))

During handling of the above exception, another exception occurred:

ConnectionError Traceback (most recent call last)
Cell In[4], line 1
----> 1 asf.download_urls(urls=urls, path='D:/spenfarm_scenes_2021_2023/spenfarm_data', session=session)

File c:\\Users\\ttgvytft90\\AppData\\Local\\anaconda3\\Lib\\site-packages\\asf_search\\download\\download.py:39, in download_urls(urls, path, session, processes)
37 if processes <= 1:
38 for url in urls:
---> 39 download_url(url=url, path=path, session=session)
40 else:
41 pool = Pool(processes=processes)

File c:\\Users\\ttgvytft90\\AppData\\Local\\anaconda3\\Lib\\site-packages\\asf_search\\download\\download.py:72, in download_url(url, path, filename, session)
69 if session is None:
70 session = ASFSession()
---> 72 response = _try_get_response(session=session, url=url)
74 with open(os.path.join(path, filename), 'wb') as f:
75 for chunk in response.iter_content(chunk_size=8192):

File c:\\Users\\ttgvytft90\\AppData\\Local\\anaconda3\\Lib\\site-packages\\tenacity\\__init__.py:289, in BaseRetrying.wraps.<locals>.wrapped_f(*args, **kw)
287 @functools.wraps(f)
288 def wrapped_f(*args: t.Any, **kw: t.Any) -> t.Any:
--> 289 return self(f, *args, **kw)

File c:\\Users\\ttgvytft90\\AppData\\Local\\anaconda3\\Lib\\site-packages\\tenacity\\__init__.py:379, in Retrying.__call__(self, fn, *args, **kwargs)
377 retry_state = RetryCallState(retry_object=self, fn=fn, args=args, kwargs=kwargs)
378 while True:
--> 379 do = self.iter(retry_state=retry_state)
380 if isinstance(do, DoAttempt):
381 try:

File c:\\Users\\ttgvytft90\\AppData\\Local\\anaconda3\\Lib\\site-packages\\tenacity\\__init__.py:314, in BaseRetrying.iter(self, retry_state)
312 is_explicit_retry = fut.failed and isinstance(fut.exception(), TryAgain)
313 if not (is_explicit_retry or self.retry(retry_state)):
--> 314 return fut.result()
316 if self.after is not None:
317 self.after(retry_state)

File c:\\Users\\ttgvytft90\\AppData\\Local\\anaconda3\\Lib\\concurrent\\futures\\_base.py:449, in Future.result(self, timeout)
447 raise CancelledError()
448 elif self._state == FINISHED:
--> 449 return self.__get_result()
451 self._condition.wait(timeout)
453 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:

File c:\\Users\\ttgvytft90\\AppData\\Local\\anaconda3\\Lib\\concurrent\\futures\\_base.py:401, in Future.__get_result(self)
399 if self._exception:
400 try:
--> 401 raise self._exception
402 finally:
403 # Break a reference cycle with the exception in self._exception
404 self = None

File c:\\Users\\ttgvytft90\\AppData\\Local\\anaconda3\\Lib\\site-packages\\tenacity\\__init__.py:382, in Retrying.__call__(self, fn, *args, **kwargs)
380 if isinstance(do, DoAttempt):
381 try:
--> 382 result = fn(*args, **kwargs)
383 except BaseException: # noqa: B902
384 retry_state.set_exception(sys.exc_info()) # type: ignore[arg-type]

File c:\\Users\\ttgvytft90\\AppData\\Local\\anaconda3\\Lib\\site-packages\\asf_search\\download\\download.py:120, in _try_get_response(session, url)
113 @retry(
114 reraise=True,
115 retry=retry_if_result(_is_burst_processing),
(...)
118 )
119 def _try_get_response(session: ASFSession, url: str):
--> 120 response = session.get(url, stream=True, hooks={'response': strip_auth_if_aws})
122 try:
123 response.raise_for_status()

File c:\\Users\\ttgvytft90\\AppData\\Local\\anaconda3\\Lib\\site-packages\\requests\\sessions.py:602, in Session.get(self, url, **kwargs)
594 r\"\"\"Sends a GET request. Returns :class:Response object.
595
596 :param url: URL for the new :class:Request object.
597 :param \\*\\*kwargs: Optional arguments that `request takes.
598 :rtype: requests.Response
599 \"\"\"
601 kwargs.setdefault(\"allow_redirects\", True)
--> 602 return self.request(\"GET\", url, **kwargs)

File c:\\Users\\ttgvytft90\\AppData\\Local\\anaconda3\\Lib\\site-packages\\requests\\sessions.py:589, in Session.request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
584 send_kwargs = {
585 \"timeout\": timeout,
586 \"allow_redirects\": allow_redirects,
587 }
588 send_kwargs.update(settings)
--> 589 resp = self.send(prep, **send_kwargs)
591 return resp

File c:\\Users\\ttgvytft90\\AppData\\Local\\anaconda3\\Lib\\site-packages\\requests\\sessions.py:703, in Session.send(self, request, **kwargs)
700 start = preferred_clock()
702 # Send the request
--> 703 r = adapter.send(request, **kwargs)
705 # Total elapsed time of the request (approximately)
706 elapsed = preferred_clock() - start

File c:\\Users\\ttgvytft90\\AppData\\Local\\anaconda3\\Lib\\site-packages\\requests\\adapters.py:682, in HTTPAdapter.send(self, request, stream, timeout, verify, cert, proxies)
667 resp = conn.urlopen(
668 method=request.method,
669 url=url,
(...)
678 chunked=chunked,
679 )
681 except (ProtocolError, OSError) as err:
--> 682 raise ConnectionError(err, request=request)
684 except MaxRetryError as e:
685 if isinstance(e.reason, ConnectTimeoutError):
686 # TODO: Remove this in 3.0.0: see #2811

ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))"
}

Hello,

I am facing the same error since yesterday a 9pm approx (Spain schedule):

"ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))"

It worked fine during more than 5 hours until the time I previously said.
I am downloading this dataset (https://disc.gsfc.nasa.gov/datasets/GLDAS_CLSM025_DA1_D_2.2/summary?keywords=GLDAS) for a bounding box localized over South Africa.
I get to retrieve a few daily datasets but after 1 min of downloading the error appears again and again.
You can find attached the error-log as a pdf.
Thank you in advance.
Miguel

Hello Rhoda Mcharo,
It looks like you are trying to use your own script to download Sentinel-1 data. Is this correct? If so, this will not work because you cannot create and manage the necessary download session where user credentials are passed to the NASA authentication service. You must use asf_search (https://docs.asf.alaska.edu/asf_search/basics/) or the Search API (https://docs.asf.alaska.edu/api/basics/) to programmatically download data from ASF.

Thank you.

Hello Miguel González Jiménez,
The GLDAS dataset is not available from the ASF DAAC. Please submit a new Earthdata Forum ticket and tag it with "GES DISC" instead of ASF.

Thank you.