Commit a728e126 authored by aknecht2's avatar aknecht2
Browse files

Revert "Updated download_fastq to use 4096b chunks."

This reverts commit 1c068ff3.
parent 1c068ff3
import sys
import hashlib
import urllib2
import os
import traceback
#!/usr/bin/python
import chipathlon.utils
import argparse
parser = argparse.ArgumentParser(description="Download target file.")
parser.add_argument("-u", "--url", dest="url", required=True, help="Target url.")
parser.add_argument("-p", "--path", dest="path", required=True, help="Local path to file.")
parser.add_argument("-t", "--url_type", dest="url_type", default="ftp://", help="Type of url to access.")
parser.add_argument("-r", "--retries", dest="retries", default=3, type=int, help="Number of retries.")
parser.add_argument("-n", "--overwrite", dest="overwrite", default=True, action="store_false", help="Dont' overwrite local file if exists.")
parser.add_argument("-m", "--md5", dest="md5", help="Check md5 value against passed value.")
args = parser.parse_args()
def progress(current, end, length=20):
percent = float(current) / end
hashes = "#" * int(round(percent * length))
spaces = " " * (length - len(hashes))
sys.stdout.write("\rProcessed %s / %s entries. [%s] %s%%" % (current, end, hashes + spaces, int(round(percent * 100))))
sys.stdout.flush()
def downloadFile(url, localpath, urltype="ftp://", retries=3, overwrite=True, checkmd5=False, md5=None):
success = False
if url[:7] == "http://":
urltype = ""
if not os.path.isfile(localpath) or overwrite:
for i in range(0, retries):
print "Attempt #%s, downloading %s\n" % (i + 1, url)
try:
hash_md5 = hashlib.md5()
conn = urllib2.urlopen(urltype + url)
with open(localpath, "w") as fwh:
for chunk in iter(lambda: conn.read(4 * 1024), ""):
fwh.write(chunk)
hash_md5.update(chunk)
if checkmd5:
if hash_md5.hexdigest() == md5:
success = True
print "File saved successfully.\n"
else:
print "MD5 mismatch (input=[%s], hexdigest=[%s]). Retrying.\n" % (md5, hashlib.md5(data).hexdigest())
else:
success = True
print "File saved successfully.\n"
except urllib2.URLError as e:
print(traceback.format_exc())
overwrite = True
finally:
if conn:
conn.close()
if success:
return
else:
print "File already exists, skipping download.\n"
return
chipathlon.utils.downloadFile(
args.url,
args.path,
urltype=args.url_type,
retries=args.retries,
overwrite=args.overwrite,
checkmd5=(not not args.md5),
md5=args.md5
)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment