diff --git a/examples/workflows/alexmac/imgproc.json b/examples/workflows/alexmac/imgproc.json
index d73feb4e9a2bf1535609d838ab3dd5609b6d7aeb..0c475418e08a17d1801920391eb91fc45e8ad9a2 100644
--- a/examples/workflows/alexmac/imgproc.json
+++ b/examples/workflows/alexmac/imgproc.json
@@ -24,7 +24,7 @@
"depends": ["gray"]
},
{
- "name": "thresh",
+ "name": "threshold",
"executable": "ih-threshold",
"inputs": ["blurred"],
"outputs": ["masked"],
@@ -41,7 +41,7 @@
"inputs": ["masked", "base"],
"outputs": ["recolor"],
"arguments": {},
- "depends": ["thresh"]
+ "depends": ["threshold"]
},
{
"name": "crop",
@@ -107,7 +107,7 @@
"depends": ["crop"]
},
{
- "name": "bgr_lab",
+ "name": "bgr2lab",
"executable": "ih-convert-color",
"inputs": ["cf1"],
"outputs": ["lab_spectrum"],
@@ -125,7 +125,7 @@
"arguments": {
"--logic": "((r - g) > 5)"
},
- "depends": ["bgr_lab"]
+ "depends": ["bgr2lab"]
},
{
"name": "labfilter2",
@@ -159,7 +159,7 @@
"inputs": ["recolor"],
"outputs": ["cf2"],
"arguments": {
- "--logic": "(((r - g) < ((i * i) / 4000)) or (((r - g) > (40 * (1 - (i / 765)))) and ((r - b) > (60 * (1 - (i / 765))))))"
+ "--logic": "(((r - g) < ((i . i) / 4000)) or (((r - g) > (40 . (1 - (i / 765)))) and ((r - b) > (60 . (1 - (i / 765))))))"
},
"depends": ["reconstitute"]
},
@@ -301,7 +301,7 @@
]
},
"options": {
-
+ "save-steps": "true"
},
"extract": {
"histogram-bin": {
diff --git a/examples/workflows/alexmac/output/fluo_hist_bins.json b/examples/workflows/alexmac/output/fluo_hist_bins.json
new file mode 100644
index 0000000000000000000000000000000000000000..42098f96dcdef5c640f4a32ac21ff1e07be4fe04
--- /dev/null
+++ b/examples/workflows/alexmac/output/fluo_hist_bins.json
@@ -0,0 +1 @@
+[{"max": [255, 255, 54], "name": "fluo_bin1", "min": [1, 1, 1]}, {"max": [255, 255, 59], "name": "fluo_bin2", "min": [1, 1, 54]}, {"max": [255, 255, 64], "name": "fluo_bin3", "min": [1, 1, 59]}, {"max": [255, 255, 70], "name": "fluo_bin4", "min": [1, 1, 64]}, {"max": [255, 255, 76], "name": "fluo_bin5", "min": [1, 1, 70]}, {"max": [255, 255, 83], "name": "fluo_bin6", "min": [1, 1, 76]}, {"max": [255, 255, 91], "name": "fluo_bin7", "min": [1, 1, 83]}, {"max": [255, 255, 101], "name": "fluo_bin8", "min": [1, 1, 91]}, {"max": [255, 255, 121], "name": "fluo_bin9", "min": [1, 1, 101]}, {"max": [255, 255, 255], "name": "fluo_bin10", "min": [1, 1, 121]}]
\ No newline at end of file
diff --git a/examples/workflows/alexmac/output/images.csv b/examples/workflows/alexmac/output/images.csv
new file mode 100644
index 0000000000000000000000000000000000000000..6afa5bb38b91e82a15fa82f17b955b087684ef7d
Binary files /dev/null and b/examples/workflows/alexmac/output/images.csv differ
diff --git a/examples/workflows/alexmac/output/img3.db b/examples/workflows/alexmac/output/img3.db
new file mode 100644
index 0000000000000000000000000000000000000000..19c3c05a317bdfdb2210874587992d9e18cee7d7
Binary files /dev/null and b/examples/workflows/alexmac/output/img3.db differ
diff --git a/examples/workflows/alexmac/output/imgproc.log b/examples/workflows/alexmac/output/imgproc.log
new file mode 100644
index 0000000000000000000000000000000000000000..ab096d8d15aa8c9df43563abd4765c2dd626f6ea
--- /dev/null
+++ b/examples/workflows/alexmac/output/imgproc.log
@@ -0,0 +1,3 @@
+Image processing error log
+==========================
+Image with id 'fluosv191' loaded from input path '/home/aknecht/stash/9-22-15_lemnatec_images/alexmac/81715 alexmac_81715-004D_2015-08-17_03-07-09_1855600/Fluo_SV_0_2/0_0_0.png' was not processed successfully.
diff --git a/examples/workflows/alexmac/output/output.db b/examples/workflows/alexmac/output/output.db
new file mode 100644
index 0000000000000000000000000000000000000000..3996eb7ee61298764a1c86c7320fb7329f1393c9
Binary files /dev/null and b/examples/workflows/alexmac/output/output.db differ
diff --git a/examples/workflows/alexmac/output/rgb_hist_bins.json b/examples/workflows/alexmac/output/rgb_hist_bins.json
new file mode 100644
index 0000000000000000000000000000000000000000..8256d25baa559c7fbed8b3926937aa01a15603d6
--- /dev/null
+++ b/examples/workflows/alexmac/output/rgb_hist_bins.json
@@ -0,0 +1 @@
+[{"max": [30, 63, 57], "name": "rgb_bin1", "min": [1, 1, 1]}, {"max": [30, 63, 75], "name": "rgb_bin2", "min": [1, 1, 57]}, {"max": [30, 63, 100], "name": "rgb_bin3", "min": [1, 1, 75]}, {"max": [30, 63, 137], "name": "rgb_bin4", "min": [1, 1, 100]}, {"max": [30, 63, 255], "name": "rgb_bin5", "min": [1, 1, 137]}, {"max": [30, 78, 57], "name": "rgb_bin6", "min": [1, 63, 1]}, {"max": [30, 78, 75], "name": "rgb_bin7", "min": [1, 63, 57]}, {"max": [30, 78, 100], "name": "rgb_bin8", "min": [1, 63, 75]}, {"max": [30, 78, 137], "name": "rgb_bin9", "min": [1, 63, 100]}, {"max": [30, 78, 255], "name": "rgb_bin10", "min": [1, 63, 137]}, {"max": [30, 102, 57], "name": "rgb_bin11", "min": [1, 78, 1]}, {"max": [30, 102, 75], "name": "rgb_bin12", "min": [1, 78, 57]}, {"max": [30, 102, 100], "name": "rgb_bin13", "min": [1, 78, 75]}, {"max": [30, 102, 137], "name": "rgb_bin14", "min": [1, 78, 100]}, {"max": [30, 102, 255], "name": "rgb_bin15", "min": [1, 78, 137]}, {"max": [30, 139, 57], "name": "rgb_bin16", "min": [1, 102, 1]}, {"max": [30, 139, 75], "name": "rgb_bin17", "min": [1, 102, 57]}, {"max": [30, 139, 100], "name": "rgb_bin18", "min": [1, 102, 75]}, {"max": [30, 139, 137], "name": "rgb_bin19", "min": [1, 102, 100]}, {"max": [30, 139, 255], "name": "rgb_bin20", "min": [1, 102, 137]}, {"max": [30, 255, 57], "name": "rgb_bin21", "min": [1, 139, 1]}, {"max": [30, 255, 75], "name": "rgb_bin22", "min": [1, 139, 57]}, {"max": [30, 255, 100], "name": "rgb_bin23", "min": [1, 139, 75]}, {"max": [30, 255, 137], "name": "rgb_bin24", "min": [1, 139, 100]}, {"max": [30, 255, 255], "name": "rgb_bin25", "min": [1, 139, 137]}, {"max": [52, 63, 57], "name": "rgb_bin26", "min": [30, 1, 1]}, {"max": [52, 63, 75], "name": "rgb_bin27", "min": [30, 1, 57]}, {"max": [52, 63, 100], "name": "rgb_bin28", "min": [30, 1, 75]}, {"max": [52, 63, 137], "name": "rgb_bin29", "min": [30, 1, 100]}, {"max": [52, 63, 255], "name": "rgb_bin30", "min": [30, 1, 137]}, {"max": [52, 78, 57], "name": "rgb_bin31", "min": [30, 63, 1]}, {"max": [52, 78, 75], "name": "rgb_bin32", "min": [30, 63, 57]}, {"max": [52, 78, 100], "name": "rgb_bin33", "min": [30, 63, 75]}, {"max": [52, 78, 137], "name": "rgb_bin34", "min": [30, 63, 100]}, {"max": [52, 78, 255], "name": "rgb_bin35", "min": [30, 63, 137]}, {"max": [52, 102, 57], "name": "rgb_bin36", "min": [30, 78, 1]}, {"max": [52, 102, 75], "name": "rgb_bin37", "min": [30, 78, 57]}, {"max": [52, 102, 100], "name": "rgb_bin38", "min": [30, 78, 75]}, {"max": [52, 102, 137], "name": "rgb_bin39", "min": [30, 78, 100]}, {"max": [52, 102, 255], "name": "rgb_bin40", "min": [30, 78, 137]}, {"max": [52, 139, 57], "name": "rgb_bin41", "min": [30, 102, 1]}, {"max": [52, 139, 75], "name": "rgb_bin42", "min": [30, 102, 57]}, {"max": [52, 139, 100], "name": "rgb_bin43", "min": [30, 102, 75]}, {"max": [52, 139, 137], "name": "rgb_bin44", "min": [30, 102, 100]}, {"max": [52, 139, 255], "name": "rgb_bin45", "min": [30, 102, 137]}, {"max": [52, 255, 57], "name": "rgb_bin46", "min": [30, 139, 1]}, {"max": [52, 255, 75], "name": "rgb_bin47", "min": [30, 139, 57]}, {"max": [52, 255, 100], "name": "rgb_bin48", "min": [30, 139, 75]}, {"max": [52, 255, 137], "name": "rgb_bin49", "min": [30, 139, 100]}, {"max": [52, 255, 255], "name": "rgb_bin50", "min": [30, 139, 137]}, {"max": [86, 63, 57], "name": "rgb_bin51", "min": [52, 1, 1]}, {"max": [86, 63, 75], "name": "rgb_bin52", "min": [52, 1, 57]}, {"max": [86, 63, 100], "name": "rgb_bin53", "min": [52, 1, 75]}, {"max": [86, 63, 137], "name": "rgb_bin54", "min": [52, 1, 100]}, {"max": [86, 63, 255], "name": "rgb_bin55", "min": [52, 1, 137]}, {"max": [86, 78, 57], "name": "rgb_bin56", "min": [52, 63, 1]}, {"max": [86, 78, 75], "name": "rgb_bin57", "min": [52, 63, 57]}, {"max": [86, 78, 100], "name": "rgb_bin58", "min": [52, 63, 75]}, {"max": [86, 78, 137], "name": "rgb_bin59", "min": [52, 63, 100]}, {"max": [86, 78, 255], "name": "rgb_bin60", "min": [52, 63, 137]}, {"max": [86, 102, 57], "name": "rgb_bin61", "min": [52, 78, 1]}, {"max": [86, 102, 75], "name": "rgb_bin62", "min": [52, 78, 57]}, {"max": [86, 102, 100], "name": "rgb_bin63", "min": [52, 78, 75]}, {"max": [86, 102, 137], "name": "rgb_bin64", "min": [52, 78, 100]}, {"max": [86, 102, 255], "name": "rgb_bin65", "min": [52, 78, 137]}, {"max": [86, 139, 57], "name": "rgb_bin66", "min": [52, 102, 1]}, {"max": [86, 139, 75], "name": "rgb_bin67", "min": [52, 102, 57]}, {"max": [86, 139, 100], "name": "rgb_bin68", "min": [52, 102, 75]}, {"max": [86, 139, 137], "name": "rgb_bin69", "min": [52, 102, 100]}, {"max": [86, 139, 255], "name": "rgb_bin70", "min": [52, 102, 137]}, {"max": [86, 255, 57], "name": "rgb_bin71", "min": [52, 139, 1]}, {"max": [86, 255, 75], "name": "rgb_bin72", "min": [52, 139, 57]}, {"max": [86, 255, 100], "name": "rgb_bin73", "min": [52, 139, 75]}, {"max": [86, 255, 137], "name": "rgb_bin74", "min": [52, 139, 100]}, {"max": [86, 255, 255], "name": "rgb_bin75", "min": [52, 139, 137]}, {"max": [131, 63, 57], "name": "rgb_bin76", "min": [86, 1, 1]}, {"max": [131, 63, 75], "name": "rgb_bin77", "min": [86, 1, 57]}, {"max": [131, 63, 100], "name": "rgb_bin78", "min": [86, 1, 75]}, {"max": [131, 63, 137], "name": "rgb_bin79", "min": [86, 1, 100]}, {"max": [131, 63, 255], "name": "rgb_bin80", "min": [86, 1, 137]}, {"max": [131, 78, 57], "name": "rgb_bin81", "min": [86, 63, 1]}, {"max": [131, 78, 75], "name": "rgb_bin82", "min": [86, 63, 57]}, {"max": [131, 78, 100], "name": "rgb_bin83", "min": [86, 63, 75]}, {"max": [131, 78, 137], "name": "rgb_bin84", "min": [86, 63, 100]}, {"max": [131, 78, 255], "name": "rgb_bin85", "min": [86, 63, 137]}, {"max": [131, 102, 57], "name": "rgb_bin86", "min": [86, 78, 1]}, {"max": [131, 102, 75], "name": "rgb_bin87", "min": [86, 78, 57]}, {"max": [131, 102, 100], "name": "rgb_bin88", "min": [86, 78, 75]}, {"max": [131, 102, 137], "name": "rgb_bin89", "min": [86, 78, 100]}, {"max": [131, 102, 255], "name": "rgb_bin90", "min": [86, 78, 137]}, {"max": [131, 139, 57], "name": "rgb_bin91", "min": [86, 102, 1]}, {"max": [131, 139, 75], "name": "rgb_bin92", "min": [86, 102, 57]}, {"max": [131, 139, 100], "name": "rgb_bin93", "min": [86, 102, 75]}, {"max": [131, 139, 137], "name": "rgb_bin94", "min": [86, 102, 100]}, {"max": [131, 139, 255], "name": "rgb_bin95", "min": [86, 102, 137]}, {"max": [131, 255, 57], "name": "rgb_bin96", "min": [86, 139, 1]}, {"max": [131, 255, 75], "name": "rgb_bin97", "min": [86, 139, 57]}, {"max": [131, 255, 100], "name": "rgb_bin98", "min": [86, 139, 75]}, {"max": [131, 255, 137], "name": "rgb_bin99", "min": [86, 139, 100]}, {"max": [131, 255, 255], "name": "rgb_bin100", "min": [86, 139, 137]}, {"max": [255, 63, 57], "name": "rgb_bin101", "min": [131, 1, 1]}, {"max": [255, 63, 75], "name": "rgb_bin102", "min": [131, 1, 57]}, {"max": [255, 63, 100], "name": "rgb_bin103", "min": [131, 1, 75]}, {"max": [255, 63, 137], "name": "rgb_bin104", "min": [131, 1, 100]}, {"max": [255, 63, 255], "name": "rgb_bin105", "min": [131, 1, 137]}, {"max": [255, 78, 57], "name": "rgb_bin106", "min": [131, 63, 1]}, {"max": [255, 78, 75], "name": "rgb_bin107", "min": [131, 63, 57]}, {"max": [255, 78, 100], "name": "rgb_bin108", "min": [131, 63, 75]}, {"max": [255, 78, 137], "name": "rgb_bin109", "min": [131, 63, 100]}, {"max": [255, 78, 255], "name": "rgb_bin110", "min": [131, 63, 137]}, {"max": [255, 102, 57], "name": "rgb_bin111", "min": [131, 78, 1]}, {"max": [255, 102, 75], "name": "rgb_bin112", "min": [131, 78, 57]}, {"max": [255, 102, 100], "name": "rgb_bin113", "min": [131, 78, 75]}, {"max": [255, 102, 137], "name": "rgb_bin114", "min": [131, 78, 100]}, {"max": [255, 102, 255], "name": "rgb_bin115", "min": [131, 78, 137]}, {"max": [255, 139, 57], "name": "rgb_bin116", "min": [131, 102, 1]}, {"max": [255, 139, 75], "name": "rgb_bin117", "min": [131, 102, 57]}, {"max": [255, 139, 100], "name": "rgb_bin118", "min": [131, 102, 75]}, {"max": [255, 139, 137], "name": "rgb_bin119", "min": [131, 102, 100]}, {"max": [255, 139, 255], "name": "rgb_bin120", "min": [131, 102, 137]}, {"max": [255, 255, 57], "name": "rgb_bin121", "min": [131, 139, 1]}, {"max": [255, 255, 75], "name": "rgb_bin122", "min": [131, 139, 57]}, {"max": [255, 255, 100], "name": "rgb_bin123", "min": [131, 139, 75]}, {"max": [255, 255, 137], "name": "rgb_bin124", "min": [131, 139, 100]}, {"max": [255, 255, 255], "name": "rgb_bin125", "min": [131, 139, 137]}]
\ No newline at end of file
diff --git a/examples/workflows/alexmac/stats.json b/examples/workflows/alexmac/stats.json
new file mode 100644
index 0000000000000000000000000000000000000000..ee72e6a9b721f90e52bf37f6e2fb160cc48520e8
--- /dev/null
+++ b/examples/workflows/alexmac/stats.json
@@ -0,0 +1,36 @@
+{
+ "rgb": [
+ {
+ "name": "area",
+ "executable": "pcv-stats-shoot-area",
+ "inputs": ["images"],
+ "outputs": ["shootarea"],
+ "arguments": {
+ "--group": {
+ "rgb": ["rgbsv"]
+ }
+ },
+ "depends": ["bin"]
+ },
+ {
+ "name": "normal",
+ "executable": "pcv-stats-normalize",
+ "inputs": ["shootarea"],
+ "outputs": ["normal"],
+ "arguments": {
+ "--column": "pixels"
+ },
+ "depends": ["area"]
+ },
+ {
+ "name": "threshold",
+ "executable": "pcv-stats-threshold",
+ "inputs": ["normal"],
+ "outputs": ["thresh"],
+ "arguments": {
+ "--thresh": 0.01
+ },
+ "depends": ["normal"]
+ }
+ ]
+}
diff --git a/ih/workflow.py b/ih/workflow.py
index 20a79f6d0e2ba4c70e1512bc8100340c76fa0526..a5b922e2f0a6e1d5abd869d73ef1da316600ef1c 100644
--- a/ih/workflow.py
+++ b/ih/workflow.py
@@ -383,16 +383,17 @@ class Statistics(Workflow):
Creates the pegasus submit script. submit.sh
"""
with open(self.basepath + "/" + loc + "/submit.sh", "w") as wh:
+
submit = textwrap.dedent("""\
#!/bin/bash
-
+ %s
plan=`pegasus-plan \\
--conf "%s" \\
--sites "%s" \\
--dir "%s" \\
--output-site local \\
--dax "%s" \\
- --randomdir \\""" % (self.basepath + "/" + loc + "/conf.rc", "condorpool" if "osg" in self.config else "local", self.basepath + "/work/stats", self.basepath + "/" + loc + "/stats.dax"))
+ --randomdir \\""" % ("module unload python/2.7" if "osg" in self.config else "", self.basepath + "/" + loc + "/conf.rc", "condorpool" if "osg" in self.config else "local", self.basepath + "/work/stats", self.basepath + "/" + loc + "/stats.dax"))
if "cluster" in self.config:
submit += """--cluster horizontal \\\n"""
submit += textwrap.dedent("""\
@@ -693,8 +694,8 @@ class ImageProcessor(Workflow):
arguments["--createdb"] = ""
arguments["--inputs"] = " ".join([x for x in exInput[type][q].keys() if ".png" in x])
self._addFile(type + str(q) + ".db", type, "output")
- self._addJob(type + "_extract" + str(q), "ih-extract-multi", exInput[type][q], {"db": {"file": type + str(q) + ".db", "transfer": False}}, arguments, exDep[type][q], walltime = 240)
- self._addJob(type + "_extract" + str(q), "ih-extract-multi", exInput[type][q], {"db": {"file": type + str(q) + ".db", "transfer": False}}, arguments, [], dax = self.exdax, walltime = 240)
+ self._addJob(type + "_extract" + str(q), "ih-extract-multi", exInput[type][q], {"db": {"file": type + str(q) + ".db", "transfer": False}}, arguments, exDep[type][q], walltime = 300)
+ self._addJob(type + "_extract" + str(q), "ih-extract-multi", exInput[type][q], {"db": {"file": type + str(q) + ".db", "transfer": False}}, arguments, [], dax = self.exdax, walltime = 300)
maprc.write(type + str(q) + ".db" + " file://" + self.basepath + "/output/" + type + str(q) + ".db" + " pool=\"local\"\n")
binDep.append(type + "_extract" + str(q))
aggIn[type + str(q) + ".db"] = {"file": type + str(q) + ".db", "transfer": False}
@@ -726,8 +727,8 @@ class ImageProcessor(Workflow):
exInput[type][q]["db"] = {"file": type + str(q) + ".db", "transfer": False}
exInput[type][q]["binfile"] = {"file": map[type] + "_hist_bins.json", "transfer": False}
arguments["--bins"] = "binfile"
- self._addJob(type + "_extractBins" + str(q), "ih-extract-multi", exInput[type][q], {}, arguments, ["bin_creation"], walltime = 240)
- self._addJob(type + "_extractBins" + str(q), "ih-extract-multi", exInput[type][q], {}, arguments, ["bin_creation"], dax = self.exdax, walltime = 240)
+ self._addJob(type + "_extractBins" + str(q), "ih-extract-multi", exInput[type][q], {}, arguments, ["bin_creation"], walltime = 480)
+ self._addJob(type + "_extractBins" + str(q), "ih-extract-multi", exInput[type][q], {}, arguments, ["bin_creation"], dax = self.exdax, walltime = 480)
binDep.append(type + "_extractBins" + str(q))
aggIn["db"] = {"file": "img2.db", "transfer": False}
self._addJob("sql_aggregate2", "ih-sql-aggregate", aggIn, {"img3.db": {"file": "img3.db", "transfer": True}}, {"--db": "db", "--output": "img3.db", "--inputs": " ".join([aggIn[x]["file"] for x in aggIn if x != "db"])}, binDep)