Commit 855abaaf authored by aknecht2's avatar aknecht2
Browse files

Partition extraction jobs into smaller segments. Added dagman.maxjobs...

Partition extraction jobs into smaller segments.  Added dagman.maxjobs definition for osg workflows.
parent cce2a4e9
......@@ -648,11 +648,11 @@ class ImageProcessor(Workflow):
if jobnum == self.config["cluster"]:
jobnum = 0
clusternum[type] += 1
exDep[type].append([])
exInput[type].append({})
if ((clusternum[type] * 100 + jobnum) % int(self.config["cluster"] * 0.3)) == 0:
meancluster[type] += 1
if (jobnum % 50) == 0:
exDep[type].append([])
exInput[type].append({})
excluster[type] += 1
extension = "." + infile.split(".")[1]
realname = self.files[self.dax][type]["input"][infile]["path"].split("/")[-1].split(".")[0]
......@@ -783,6 +783,8 @@ class ImageProcessor(Workflow):
pegasus.stageout.clusters = 4
pegasus.transfer.threads = 4
pegasus.transfer.lite.threads = 4
dagman.maxjobs = 200
""")
wh.write(pegasusrc)
return
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment