diff --git a/chipathlon/db.py b/chipathlon/db.py index cbb25e72731f074aed3c85619af15c754afcf2ac..4903f77e0b7f3740493096a6b02b27f999622f6c 100644 --- a/chipathlon/db.py +++ b/chipathlon/db.py @@ -11,7 +11,20 @@ import chipathlon.conf from pprint import pprint import hashlib from chipathlon.utils import progress - +import bson + +def download_from_gridfs(host, gridfs_id, local_path, username=None, password=None, retries=3, overwrite=True, checkmd5=False): + mdb = chipathlon.db.MongoDB(host, username, password) + if not os.path.isfile(local_path) or overwrite: + for i in range(0, retries): + print "Attempt #%s, downloading file with ID '%s' to '%s'" % (i + 1, gridfs_id, local_path) + if mdb.fetch_from_gridfs(bson.objectid.ObjectId(gridfs_id), localpath, checkmd5): + return True + else: + print "Download attempt #%s from GridFS failed, retrying..." % (i + 1) + else: + print "File already exists, skipping download.\n" + return False class MongoDB(object): """ diff --git a/chipathlon/utils.py b/chipathlon/utils.py index 33176caa87740b554a9df93e15f6ef04e5ac969a..83419471360a648b955966c89b9e7eeabef043d1 100644 --- a/chipathlon/utils.py +++ b/chipathlon/utils.py @@ -3,8 +3,6 @@ import hashlib import urllib2 import os import traceback -import chipathlon.db -import bson.objectid def progress(current, end, length=20): percent = float(current) / end @@ -50,21 +48,6 @@ def downloadFile(url, localpath, urltype="http://", retries=3, overwrite=True, c print "File already exists, skipping download.\n" return -def downloadFromGridFS(hostname, username, password, gridfs_id, localpath, retries=3, overwrite=True, checkmd5=False): - success = False - mdb = chipathlon.db.MongoDB(hostname, username, password) - if not os.path.isfile(localpath) or overwrite: - for i in range(0, retries): - print "Attempt #%s, downloading file with ID '%s' to '%s'" % (i + 1, gridfs_id, localpath) - if mdb.fetch_from_gridfs(bson.objectid.ObjectId(gridfs_id), localpath, checkmd5): - success = True - break - else: - print "Download attempt #%s from GridFS failed, retrying..." % (i + 1) - else: - print "File already exists, skipping download.\n" - return success - # http://pythoncentral.io/how-to-check-if-a-string-is-a-number-in-python-including-unicode/ def is_number(s): try: diff --git a/scripts/chip-job-download-gridfs b/scripts/chip-job-download-gridfs index 97e59f97c9739679b7567b37a4605fb6f91e1406..f10efb1a3bfc7dc33ba930c2e159785c9063b244 100755 --- a/scripts/chip-job-download-gridfs +++ b/scripts/chip-job-download-gridfs @@ -1,5 +1,5 @@ #!/usr/bin/env python -import chipathlon.utils +import chipathlon.db import argparse parser = argparse.ArgumentParser(description="Download target file from GridFS.") @@ -9,17 +9,17 @@ parser.add_argument("-p", "--password", dest="password", help="Database password parser.add_argument("-i", "--id", dest="gridfs_id", required=True, help="GridFS ID.") parser.add_argument("-d", "--destination", dest="destination", required=True, help="Local path to file destination.") parser.add_argument("-r", "--retries", dest="retries", default=3, type=int, help="Number of retries.") -parser.add_argument("-n", "--overwrite", dest="overwrite", default=True, action="store_false", help="Don't overwrite local file if it exists.") +parser.add_argument("-n", "--no-overwrite", dest="overwrite", default=True, action="store_false", help="Don't overwrite local file if it exists.") parser.add_argument("-c", "--checkmd5", dest="checkmd5", action="store_true", help="Check md5 value of downloaded file against database value.") args = parser.parse_args() -chipathlon.utils.downloadFromGridFS( +chipathlon.db.download_from_gridfs( args.host, - args.username, - args.password, args.gridfs_id, args.destination, - args.retries, - args.overwrite, - args.checkmd5 + username=args.username, + password=args.password, + retries=args.retries, + overwrite=args.overwrite, + checkmd5=args.checkmd5 )