diff --git a/scripts/chip-meta-download b/scripts/chip-meta-download index 7c1a674c70f8809ed6f3b1635b5fbede57247266..03657fefbaa5dd2f2b51b4372d69d105fc409440 100644 --- a/scripts/chip-meta-download +++ b/scripts/chip-meta-download @@ -13,6 +13,8 @@ import datetime parser = argparse.ArgumentParser(description="Download raw JSON for all experiments.") parser.add_argument("-o", "--output-dir", dest="outputdir", default=os.getcwd(), help="Output directory. (default: %(default)s)") parser.add_argument("-q", "--quiet", action='store_true', help="Quiet mode. Do not print progress information. (default: false)") +parser.add_argument("-r", "--resume", action='store_false', \ + help="Skip re-fetching existing experiment JSON files to speed up overall download. (default: true)") args = parser.parse_args() encode_baseurl = "https://www.encodeproject.org/experiments/" @@ -37,9 +39,17 @@ if not os.path.isdir(os.path.join(args.outputdir, "data")): total = len(exp_ids) for i, exp_id in enumerate(exp_ids): exp_url = urlparse.urljoin(encode_baseurl, exp_id) - r = requests.get(exp_url, params=json_arg) - with open(os.path.join(args.outputdir, "data", "%s.json" % (exp_id,)), "w") as wh: - wh.write(r.text) + json_file = os.path.join(args.outputdir, "data", "%s.json" % (exp_id,)) + if args.resume: + if not os.path.isfile(json_file) or os.path.getsize(json_file) == 0: + r = requests.get(exp_url, params=json_arg) + with open(json_file, "w") as wh: + wh.write(r.text) + else: + r = requests.get(exp_url, params=json_arg) + with open(json_file, "w") as wh: + wh.write(r.text) + if not args.quiet: progress(i, total) if not args.quiet: