Commit 0db7bf1f authored by aknecht2's avatar aknecht2
Browse files

Adjusted download to use script instead of pegasus-transfer.

parent 6a8ae431
......@@ -102,11 +102,13 @@ class Workflow(object):
self.dax.addExecutable(self.executables[f])
break
# Overwrite pegasus::transfer to request more walltime
if config["profile"]["env"]["PEGASUS_HOME"]:
"""
if self.config["profile"]["env"]["PEGASUS_HOME"]:
self.executables["pegasus::transfer"] = Executable(name = "pegasus::transfer", os=os_type, arch=arch)
self.executables["pegasus::transfer"].addPFN(PFN("file://%s" % (os.path.join(config["profile"]["env"]["PEGASUS_HOME"]), "bin", "pegasus-transfer")
self.executables["pegasus::transfer"].addPFN(PFN("file://%s" % (os.path.join(self.config["profile"]["env"]["PEGASUS_HOME"], "bin", "pegasus-transfer"),), "local"))
self.executables["pegasus::transfer"].profile("globus", "maxwalltime", 180)
self.dax.addExecutable(self.executables["pegasus::transfer"])
"""
self._raise()
return
......@@ -280,7 +282,17 @@ class Workflow(object):
for f in treatment:
name = "%s_%s.fastq.gz" % (run["experiment"], f["accession"])
if name not in self.files:
self._add_file(name, "http://" + f["url"], "dummylocal")
# USING DOWNLOAD SCRIPT
output_file = File(name)
self.files[name] = output_file
job = Job(self.executables["download_fastq.py"])
job.profile("globus", "maxwalltime", 360)
job.uses(output_file, link=Link.OUTPUT, transfer=True)
job.addArguments("-u", f["url"], "-p", output_file, "-t http://", "-m", f["md5sum"])
self.jobs[name] = job
self.dax.addJob(job)
# USING PEGASUS TRANSFER:
# self._add_file(name, "http://" + f["url"], "dummylocal")
# We additionally keep track of how many input files
# there are to increase the number of transfer
# clusters in _create_sites
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment