Commit 855abaaf authored by aknecht2's avatar aknecht2

Partition extraction jobs into smaller segments. Added dagman.maxjobs...

Partition extraction jobs into smaller segments.  Added dagman.maxjobs definition for osg workflows.
parent cce2a4e9
......@@ -648,11 +648,11 @@ class ImageProcessor(Workflow):
if jobnum == self.config["cluster"]:
jobnum = 0
clusternum[type] += 1
exDep[type].append([])
exInput[type].append({})
if ((clusternum[type] * 100 + jobnum) % int(self.config["cluster"] * 0.3)) == 0:
meancluster[type] += 1
if (jobnum % 50) == 0:
exDep[type].append([])
exInput[type].append({})
excluster[type] += 1
extension = "." + infile.split(".")[1]
realname = self.files[self.dax][type]["input"][infile]["path"].split("/")[-1].split(".")[0]
......@@ -783,6 +783,8 @@ class ImageProcessor(Workflow):
pegasus.stageout.clusters = 4
pegasus.transfer.threads = 4
pegasus.transfer.lite.threads = 4
dagman.maxjobs = 200
""")
wh.write(pegasusrc)
return
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment