Fixes for the getanc.py and modis_atteph.py login issues

[woodbri]

Hi all,I wrote two scripts to work-around the getanc.py and modis_atteph.py login issues. Offered here "as is" under public ___domain license in the hope this helps others. Please post fixes or improvements here for others to benefit.To install them move getanc.py to getanc-real.py and move modis_atteph.py to modis_atteph-real.py and then copy the appropriate script below into getanc.py or modis_atteph.py.the new script will call the realscript with '-d' option to get the filenames, then download those files and move them to where they belong.UPDATED: added support for ozone files in getanc.pyroot@B1B8B6F:/maps/nasa/tmp# cat modis_atteph.py#!/usr/bin/env python3import subprocessimport os.pathimport sysimport osdef fetchAndStore(fname):    (fpath, fn) = os.path.split(fname)    os.makedirs(fpath, exist_ok=True)    url = 'https://oceandata.sci.gsfc.nasa.gov (/cgi/getfile/) ' + fn    wget = ['wget', '-q', '--load-cookies', '~/.urs_cookies', '--save-cookies', '~/.urs_cookies', '--auth-no-challenge=on', '--keep-session-cookies', '--content-disposition', url]    out = subprocess.run(wget, stdout= subprocess.PIPE, stderr=subprocess.STDOUT)    if os.path.exists(fn):        mv = ['mv', fn, fname]        subprocess.run(mv)    elif os.path.exists(fn + '.bz2'):        mv = ['mv', fn + '.bz2', fname + '.bz2']        subprocess.run(mv)        unzip = ['bunzip2', fname + '.bz2']        subprocess.run(unzip)    else:        print('ERROR: failed to fetch: ' + url + "\n")        print(out.stdout.decode('utf-8'))atteph = '/u/oceandata/scripts/modis_atteph-real.py';cmd = [atteph, '-d'] + sys.argv[1:]out = subprocess.run(cmd, stdout= subprocess.PIPE, stderr=subprocess.STDOUT)lines = out.stdout.decode('utf-8').splitlines()for line in lines:    line = line.strip()    if "att1=" in line or "att2=" in line or "att3=" in line:        attfile = line[5:]        print(attfile + "\n")        if not os.path.exists(attfile):            fetchAndStore(attfile)    elif "eph1=" in line or "eph2=" in line or "eph3=" in line:        ephfile = line[5:]        print(ephfile + "\n")        if not os.path.exists(ephfile):            fetchAndStore(ephfile)root@B1B8B6F:/u/oceancolor-bin/scripts# cat getanc.py#!/usr/bin/env python3import subprocessimport os.pathimport sysimport osdef fetchAndStore(fname):    (fpath, fn) = os.path.split(fname)    os.makedirs(fpath, exist_ok=True)    url = 'https://oceandata.sci.gsfc.nasa.gov (/cgi/getfile/) ' + fn    wget = ['wget', '-q', '--load-cookies', '~/.urs_cookies', '--save-cookies', '~/.urs_cookies', '--auth-no-challenge=on', '--keep-session-cookies', '--content-disposition', url]    out = subprocess.run(wget, stdout= subprocess.PIPE, stderr=subprocess.STDOUT)    if os.path.exists(fn):        mv = ['mv', fn, fname]        subprocess.run(mv)    elif os.path.exists(fn + '.bz2'):        mv = ['mv', fn + '.bz2', fname + '.bz2']        subprocess.run(mv)        unzip = ['bunzip2', fname + '.bz2']        subprocess.run(unzip)    elif "ozone" in line:        ozfile = line[7:]        print(ozfile + "\n")        if not os.path.exists(ozfile):            fetchAndStore(ozfile)    else:        print('ERROR: failed to fetch: ' + url + "\n")        print(out.stdout.decode('utf-8'))getanc = '/u/oceandata/scripts/getanc-real.py';cmd = [getanc, '-d'] + sys.argv[1:]out = subprocess.run(cmd, stdout= subprocess.PIPE, stderr=subprocess.STDOUT)lines = out.stdout.decode('utf-8').splitlines()for line in lines:    line = line.strip()    if "icefile=" in line:        icefile = line[8:]        print(icefile + "\n")        if not os.path.exists(icefile):            fetchAndStore(icefile)    elif "met" in line:        metfile = line[5:]        print(metfile + "\n")        if not os.path.exists(metfile):            fetchAndStore(metfile)    elif "sstfile=" in line:        sstfile = line[8:]        print(sstfile + "\n")        if not os.path.exists(sstfile):            fetchAndStore(sstfile)

[pavel_babyak]

<...>
UPD: Sorry, wrong info   :confused:

[pavel_babyak]

You can make it easier.
Enough to patch ProcUtils.py to work with a .netrc

My solution:
patch

diff --git a/modules/ProcUtils.py b/modules/ProcUtils.py
index edceb41..547fe7b 100644
--- a/modules/ProcUtils.py
+++ b/modules/ProcUtils.py
@@ -6,7 +6,9 @@ SeaDAS library for commonly used functions within other python scripts
from __future__ import print_function

import sys
+import netrc

+netrc_dict=netrc.netrc()

#  ------------------ DANGER -------------------
#
@@ -98,6 +100,7 @@ def _httpdl(url, request, localpath='.', outputfilename=None, ntries=5,
         verbose - get chatty about connection issues (boolean, default False)
     """
     global ofile
+    global netrc_dict
     import os
     import re
     import socket
@@ -114,6 +117,13 @@ def _httpdl(url, request, localpath='.', outputfilename=None, ntries=5,
         os.umask(0o02)
         os.makedirs(localpath, mode=0o2775)

+    cred=netrc_dict.authenticators(url)
+    if cred is not None:
+        from base64 import b64encode
+        ( _user, _acc, _pass ) = cred
+        userAndPass = b64encode(bytes(_user + ':' + _pass, "utf-8"))
+        reqHeaders.update({ 'Authorization' : 'Basic %s' %  userAndPass })
+
     urlConn, proxy = httpinit(url, timeout=timeout, urlConn=urlConn)

     try:


[pavel_babyak]

Previous variant not working :(

Trying to use requests library, should works:


diff --git a/modules/ProcUtils.py b/modules/ProcUtils.py
index edceb41..6559339 100644
--- a/modules/ProcUtils.py
+++ b/modules/ProcUtils.py
@@ -6,7 +6,7 @@ SeaDAS library for commonly used functions within other python scripts
from __future__ import print_function

import sys
-
+import requests

#  ------------------ DANGER -------------------
#
@@ -251,9 +251,46 @@ def _httpdl(url, request, localpath='.', outputfilename=None, ntries=5,
def httpdl(url, request, localpath='.', outputfilename=None, ntries=5,
            uncompress=False, timeout=30., reqHeaders={}, verbose=False,
            reuseConn=False, urlConn=None, chunk_size=DEFAULT_CHUNK_SIZE):
-    urlConn, status = _httpdl(url, request, localpath, outputfilename, ntries,
-                              uncompress, timeout, reqHeaders, verbose,
-                              reuseConn, urlConn, chunk_size)
+
+    import requests
+    import os
+    import re
+
+    r = requests.get("https://%s%s" % (url,request), stream=True)
+
+    if r.status_code == 200 or r.status_code == 206:
+
+        if outputfilename:
+            ofile = os.path.join(localpath, outputfilename)
+        else:
+            if 'content-disposition' in r.headers:
+                ofile = os.path.join(localpath, r.headers.get('content-disposition').split('filename=')[1] )
+            else:
+                ofile = os.path.join(localpath, os.path.basename(request))
+
+        if r.status_code == 200:
+            f = open(ofile, 'wb')
+        else:
+            f = open(ofile, 'ab')
+
+        for chunk in r.iter_content(chunk_size=chunk_size):
+            if chunk: # filter out keep-alive new chunks
+                f.write(chunk)
+
+        f.flush()
+
+        if re.search(".(Z|gz|bz2)$", ofile) and uncompress:
+            compressStatus = uncompressFile(ofile)
+            if compressStatus:
+                status = compressStatus
+        else:
+            status = 0
+    else:
+        status = r.status_code
+
+#    urlConn, status = _httpdl(url, request, localpath, outputfilename, ntries,
+#                              uncompress, timeout, reqHeaders, verbose,
+#                              reuseConn, urlConn, chunk_size)
     if reuseConn:
         return (urlConn, status)
     else:


[ionutserban]

Hi Pavel,

I am not an expert in Ubuntu. Could you please briefly describe how do patch ProcUtils.py to work with a .netrc? What step should I follow, more exactly.

Thank you very much!

Best,
Ionut

[pavel_babyak]

Hi.
Patch in this thread have one bug.

I wrote a separate post with the latest version of the patch and described how to apply it.

[josl]

using `requests` is a million times better than some subprocess wget hack. Seems like switching to requests is what the maintainers implemented in the Seadas scripts.