Skip to content

I
Image Harvest
Commit 43cbafc8 authored by aknecht2's avatar aknecht2
Browse files
Options

Increased wall time request for extract-multi script. Added output for alexmac experiment.

parent 01269a68
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 74988 additions and 12 deletions
examples/workflows/alexmac/imgproc.json
View file @ 43cbafc8
......@@ -24,7 +24,7 @@
"depends": ["gray"]
},
{
"name": "thresh",
"name": "threshold",
"executable": "ih-threshold",
"inputs": ["blurred"],
"outputs": ["masked"],
......@@ -41,7 +41,7 @@
"inputs": ["masked", "base"],
"outputs": ["recolor"],
"arguments": {},
"depends": ["thresh"]
"depends": ["threshold"]
},
{
"name": "crop",
......@@ -107,7 +107,7 @@
"depends": ["crop"]
},
{
"name": "bgr_lab",
"name": "bgr2lab",
"executable": "ih-convert-color",
"inputs": ["cf1"],
"outputs": ["lab_spectrum"],
......@@ -125,7 +125,7 @@
"arguments": {
"--logic": "((r - g) > 5)"
},
"depends": ["bgr_lab"]
"depends": ["bgr2lab"]
},
{
"name": "labfilter2",
......@@ -159,7 +159,7 @@
"inputs": ["recolor"],
"outputs": ["cf2"],
"arguments": {
"--logic": "(((r - g) < ((i * i) / 4000)) or (((r - g) > (40 * (1 - (i / 765)))) and ((r - b) > (60 * (1 - (i / 765))))))"
"--logic": "(((r - g) < ((i . i) / 4000)) or (((r - g) > (40 . (1 - (i / 765)))) and ((r - b) > (60 . (1 - (i / 765))))))"
},
"depends": ["reconstitute"]
},
......@@ -301,7 +301,7 @@
]
},
"options": {
"save-steps": "true"
},
"extract": {
"histogram-bin": {
......
examples/workflows/alexmac/output/fluo_hist_bins.json 0 → 100644
View file @ 43cbafc8
[{"max": [255, 255, 54], "name": "fluo_bin1", "min": [1, 1, 1]}, {"max": [255, 255, 59], "name": "fluo_bin2", "min": [1, 1, 54]}, {"max": [255, 255, 64], "name": "fluo_bin3", "min": [1, 1, 59]}, {"max": [255, 255, 70], "name": "fluo_bin4", "min": [1, 1, 64]}, {"max": [255, 255, 76], "name": "fluo_bin5", "min": [1, 1, 70]}, {"max": [255, 255, 83], "name": "fluo_bin6", "min": [1, 1, 76]}, {"max": [255, 255, 91], "name": "fluo_bin7", "min": [1, 1, 83]}, {"max": [255, 255, 101], "name": "fluo_bin8", "min": [1, 1, 91]}, {"max": [255, 255, 121], "name": "fluo_bin9", "min": [1, 1, 101]}, {"max": [255, 255, 255], "name": "fluo_bin10", "min": [1, 1, 121]}]
\ No newline at end of file
examples/workflows/alexmac/output/images.csv 0 → 100644
View file @ 43cbafc8
This diff is collapsed.
examples/workflows/alexmac/output/imgproc.log 0 → 100644
View file @ 43cbafc8
Image processing error log
==========================
Image with id 'fluosv191' loaded from input path '/home/aknecht/stash/9-22-15_lemnatec_images/alexmac/81715 alexmac_81715-004D_2015-08-17_03-07-09_1855600/Fluo_SV_0_2/0_0_0.png' was not processed successfully.
examples/workflows/alexmac/output/rgb_hist_bins.json 0 → 100644
View file @ 43cbafc8
[{"max": [30, 63, 57], "name": "rgb_bin1", "min": [1, 1, 1]}, {"max": [30, 63, 75], "name": "rgb_bin2", "min": [1, 1, 57]}, {"max": [30, 63, 100], "name": "rgb_bin3", "min": [1, 1, 75]}, {"max": [30, 63, 137], "name": "rgb_bin4", "min": [1, 1, 100]}, {"max": [30, 63, 255], "name": "rgb_bin5", "min": [1, 1, 137]}, {"max": [30, 78, 57], "name": "rgb_bin6", "min": [1, 63, 1]}, {"max": [30, 78, 75], "name": "rgb_bin7", "min": [1, 63, 57]}, {"max": [30, 78, 100], "name": "rgb_bin8", "min": [1, 63, 75]}, {"max": [30, 78, 137], "name": "rgb_bin9", "min": [1, 63, 100]}, {"max": [30, 78, 255], "name": "rgb_bin10", "min": [1, 63, 137]}, {"max": [30, 102, 57], "name": "rgb_bin11", "min": [1, 78, 1]}, {"max": [30, 102, 75], "name": "rgb_bin12", "min": [1, 78, 57]}, {"max": [30, 102, 100], "name": "rgb_bin13", "min": [1, 78, 75]}, {"max": [30, 102, 137], "name": "rgb_bin14", "min": [1, 78, 100]}, {"max": [30, 102, 255], "name": "rgb_bin15", "min": [1, 78, 137]}, {"max": [30, 139, 57], "name": "rgb_bin16", "min": [1, 102, 1]}, {"max": [30, 139, 75], "name": "rgb_bin17", "min": [1, 102, 57]}, {"max": [30, 139, 100], "name": "rgb_bin18", "min": [1, 102, 75]}, {"max": [30, 139, 137], "name": "rgb_bin19", "min": [1, 102, 100]}, {"max": [30, 139, 255], "name": "rgb_bin20", "min": [1, 102, 137]}, {"max": [30, 255, 57], "name": "rgb_bin21", "min": [1, 139, 1]}, {"max": [30, 255, 75], "name": "rgb_bin22", "min": [1, 139, 57]}, {"max": [30, 255, 100], "name": "rgb_bin23", "min": [1, 139, 75]}, {"max": [30, 255, 137], "name": "rgb_bin24", "min": [1, 139, 100]}, {"max": [30, 255, 255], "name": "rgb_bin25", "min": [1, 139, 137]}, {"max": [52, 63, 57], "name": "rgb_bin26", "min": [30, 1, 1]}, {"max": [52, 63, 75], "name": "rgb_bin27", "min": [30, 1, 57]}, {"max": [52, 63, 100], "name": "rgb_bin28", "min": [30, 1, 75]}, {"max": [52, 63, 137], "name": "rgb_bin29", "min": [30, 1, 100]}, {"max": [52, 63, 255], "name": "rgb_bin30", "min": [30, 1, 137]}, {"max": [52, 78, 57], "name": "rgb_bin31", "min": [30, 63, 1]}, {"max": [52, 78, 75], "name": "rgb_bin32", "min": [30, 63, 57]}, {"max": [52, 78, 100], "name": "rgb_bin33", "min": [30, 63, 75]}, {"max": [52, 78, 137], "name": "rgb_bin34", "min": [30, 63, 100]}, {"max": [52, 78, 255], "name": "rgb_bin35", "min": [30, 63, 137]}, {"max": [52, 102, 57], "name": "rgb_bin36", "min": [30, 78, 1]}, {"max": [52, 102, 75], "name": "rgb_bin37", "min": [30, 78, 57]}, {"max": [52, 102, 100], "name": "rgb_bin38", "min": [30, 78, 75]}, {"max": [52, 102, 137], "name": "rgb_bin39", "min": [30, 78, 100]}, {"max": [52, 102, 255], "name": "rgb_bin40", "min": [30, 78, 137]}, {"max": [52, 139, 57], "name": "rgb_bin41", "min": [30, 102, 1]}, {"max": [52, 139, 75], "name": "rgb_bin42", "min": [30, 102, 57]}, {"max": [52, 139, 100], "name": "rgb_bin43", "min": [30, 102, 75]}, {"max": [52, 139, 137], "name": "rgb_bin44", "min": [30, 102, 100]}, {"max": [52, 139, 255], "name": "rgb_bin45", "min": [30, 102, 137]}, {"max": [52, 255, 57], "name": "rgb_bin46", "min": [30, 139, 1]}, {"max": [52, 255, 75], "name": "rgb_bin47", "min": [30, 139, 57]}, {"max": [52, 255, 100], "name": "rgb_bin48", "min": [30, 139, 75]}, {"max": [52, 255, 137], "name": "rgb_bin49", "min": [30, 139, 100]}, {"max": [52, 255, 255], "name": "rgb_bin50", "min": [30, 139, 137]}, {"max": [86, 63, 57], "name": "rgb_bin51", "min": [52, 1, 1]}, {"max": [86, 63, 75], "name": "rgb_bin52", "min": [52, 1, 57]}, {"max": [86, 63, 100], "name": "rgb_bin53", "min": [52, 1, 75]}, {"max": [86, 63, 137], "name": "rgb_bin54", "min": [52, 1, 100]}, {"max": [86, 63, 255], "name": "rgb_bin55", "min": [52, 1, 137]}, {"max": [86, 78, 57], "name": "rgb_bin56", "min": [52, 63, 1]}, {"max": [86, 78, 75], "name": "rgb_bin57", "min": [52, 63, 57]}, {"max": [86, 78, 100], "name": "rgb_bin58", "min": [52, 63, 75]}, {"max": [86, 78, 137], "name": "rgb_bin59", "min": [52, 63, 100]}, {"max": [86, 78, 255], "name": "rgb_bin60", "min": [52, 63, 137]}, {"max": [86, 102, 57], "name": "rgb_bin61", "min": [52, 78, 1]}, {"max": [86, 102, 75], "name": "rgb_bin62", "min": [52, 78, 57]}, {"max": [86, 102, 100], "name": "rgb_bin63", "min": [52, 78, 75]}, {"max": [86, 102, 137], "name": "rgb_bin64", "min": [52, 78, 100]}, {"max": [86, 102, 255], "name": "rgb_bin65", "min": [52, 78, 137]}, {"max": [86, 139, 57], "name": "rgb_bin66", "min": [52, 102, 1]}, {"max": [86, 139, 75], "name": "rgb_bin67", "min": [52, 102, 57]}, {"max": [86, 139, 100], "name": "rgb_bin68", "min": [52, 102, 75]}, {"max": [86, 139, 137], "name": "rgb_bin69", "min": [52, 102, 100]}, {"max": [86, 139, 255], "name": "rgb_bin70", "min": [52, 102, 137]}, {"max": [86, 255, 57], "name": "rgb_bin71", "min": [52, 139, 1]}, {"max": [86, 255, 75], "name": "rgb_bin72", "min": [52, 139, 57]}, {"max": [86, 255, 100], "name": "rgb_bin73", "min": [52, 139, 75]}, {"max": [86, 255, 137], "name": "rgb_bin74", "min": [52, 139, 100]}, {"max": [86, 255, 255], "name": "rgb_bin75", "min": [52, 139, 137]}, {"max": [131, 63, 57], "name": "rgb_bin76", "min": [86, 1, 1]}, {"max": [131, 63, 75], "name": "rgb_bin77", "min": [86, 1, 57]}, {"max": [131, 63, 100], "name": "rgb_bin78", "min": [86, 1, 75]}, {"max": [131, 63, 137], "name": "rgb_bin79", "min": [86, 1, 100]}, {"max": [131, 63, 255], "name": "rgb_bin80", "min": [86, 1, 137]}, {"max": [131, 78, 57], "name": "rgb_bin81", "min": [86, 63, 1]}, {"max": [131, 78, 75], "name": "rgb_bin82", "min": [86, 63, 57]}, {"max": [131, 78, 100], "name": "rgb_bin83", "min": [86, 63, 75]}, {"max": [131, 78, 137], "name": "rgb_bin84", "min": [86, 63, 100]}, {"max": [131, 78, 255], "name": "rgb_bin85", "min": [86, 63, 137]}, {"max": [131, 102, 57], "name": "rgb_bin86", "min": [86, 78, 1]}, {"max": [131, 102, 75], "name": "rgb_bin87", "min": [86, 78, 57]}, {"max": [131, 102, 100], "name": "rgb_bin88", "min": [86, 78, 75]}, {"max": [131, 102, 137], "name": "rgb_bin89", "min": [86, 78, 100]}, {"max": [131, 102, 255], "name": "rgb_bin90", "min": [86, 78, 137]}, {"max": [131, 139, 57], "name": "rgb_bin91", "min": [86, 102, 1]}, {"max": [131, 139, 75], "name": "rgb_bin92", "min": [86, 102, 57]}, {"max": [131, 139, 100], "name": "rgb_bin93", "min": [86, 102, 75]}, {"max": [131, 139, 137], "name": "rgb_bin94", "min": [86, 102, 100]}, {"max": [131, 139, 255], "name": "rgb_bin95", "min": [86, 102, 137]}, {"max": [131, 255, 57], "name": "rgb_bin96", "min": [86, 139, 1]}, {"max": [131, 255, 75], "name": "rgb_bin97", "min": [86, 139, 57]}, {"max": [131, 255, 100], "name": "rgb_bin98", "min": [86, 139, 75]}, {"max": [131, 255, 137], "name": "rgb_bin99", "min": [86, 139, 100]}, {"max": [131, 255, 255], "name": "rgb_bin100", "min": [86, 139, 137]}, {"max": [255, 63, 57], "name": "rgb_bin101", "min": [131, 1, 1]}, {"max": [255, 63, 75], "name": "rgb_bin102", "min": [131, 1, 57]}, {"max": [255, 63, 100], "name": "rgb_bin103", "min": [131, 1, 75]}, {"max": [255, 63, 137], "name": "rgb_bin104", "min": [131, 1, 100]}, {"max": [255, 63, 255], "name": "rgb_bin105", "min": [131, 1, 137]}, {"max": [255, 78, 57], "name": "rgb_bin106", "min": [131, 63, 1]}, {"max": [255, 78, 75], "name": "rgb_bin107", "min": [131, 63, 57]}, {"max": [255, 78, 100], "name": "rgb_bin108", "min": [131, 63, 75]}, {"max": [255, 78, 137], "name": "rgb_bin109", "min": [131, 63, 100]}, {"max": [255, 78, 255], "name": "rgb_bin110", "min": [131, 63, 137]}, {"max": [255, 102, 57], "name": "rgb_bin111", "min": [131, 78, 1]}, {"max": [255, 102, 75], "name": "rgb_bin112", "min": [131, 78, 57]}, {"max": [255, 102, 100], "name": "rgb_bin113", "min": [131, 78, 75]}, {"max": [255, 102, 137], "name": "rgb_bin114", "min": [131, 78, 100]}, {"max": [255, 102, 255], "name": "rgb_bin115", "min": [131, 78, 137]}, {"max": [255, 139, 57], "name": "rgb_bin116", "min": [131, 102, 1]}, {"max": [255, 139, 75], "name": "rgb_bin117", "min": [131, 102, 57]}, {"max": [255, 139, 100], "name": "rgb_bin118", "min": [131, 102, 75]}, {"max": [255, 139, 137], "name": "rgb_bin119", "min": [131, 102, 100]}, {"max": [255, 139, 255], "name": "rgb_bin120", "min": [131, 102, 137]}, {"max": [255, 255, 57], "name": "rgb_bin121", "min": [131, 139, 1]}, {"max": [255, 255, 75], "name": "rgb_bin122", "min": [131, 139, 57]}, {"max": [255, 255, 100], "name": "rgb_bin123", "min": [131, 139, 75]}, {"max": [255, 255, 137], "name": "rgb_bin124", "min": [131, 139, 100]}, {"max": [255, 255, 255], "name": "rgb_bin125", "min": [131, 139, 137]}]
\ No newline at end of file
examples/workflows/alexmac/stats.json 0 → 100644
View file @ 43cbafc8
{
"rgb": [
{
"name": "area",
"executable": "pcv-stats-shoot-area",
"inputs": ["images"],
"outputs": ["shootarea"],
"arguments": {
"--group": {
"rgb": ["rgbsv"]
}
},
"depends": ["bin"]
},
{
"name": "normal",
"executable": "pcv-stats-normalize",
"inputs": ["shootarea"],
"outputs": ["normal"],
"arguments": {
"--column": "pixels"
},
"depends": ["area"]
},
{
"name": "threshold",
"executable": "pcv-stats-threshold",
"inputs": ["normal"],
"outputs": ["thresh"],
"arguments": {
"--thresh": 0.01
},
"depends": ["normal"]
}
]
}
ih/workflow.py
View file @ 43cbafc8
......@@ -383,16 +383,17 @@ class Statistics(Workflow):
Creates the pegasus submit script. submit.sh
"""
with open(self.basepath + "/" + loc + "/submit.sh", "w") as wh:
submit = textwrap.dedent("""\
#!/bin/bash
%s
plan=`pegasus-plan \\
--conf "%s" \\
--sites "%s" \\
--dir "%s" \\
--output-site local \\
--dax "%s" \\
--randomdir \\""" % (self.basepath + "/" + loc + "/conf.rc", "condorpool" if "osg" in self.config else "local", self.basepath + "/work/stats", self.basepath + "/" + loc + "/stats.dax"))
--randomdir \\""" % ("module unload python/2.7" if "osg" in self.config else "", self.basepath + "/" + loc + "/conf.rc", "condorpool" if "osg" in self.config else "local", self.basepath + "/work/stats", self.basepath + "/" + loc + "/stats.dax"))
if "cluster" in self.config:
submit += """--cluster horizontal \\\n"""
submit += textwrap.dedent("""\
......@@ -693,8 +694,8 @@ class ImageProcessor(Workflow):
arguments["--createdb"] = ""
arguments["--inputs"] = " ".join([x for x in exInput[type][q].keys() if ".png" in x])
self._addFile(type + str(q) + ".db", type, "output")
self._addJob(type + "_extract" + str(q), "ih-extract-multi", exInput[type][q], {"db": {"file": type + str(q) + ".db", "transfer": False}}, arguments, exDep[type][q], walltime = 240)
self._addJob(type + "_extract" + str(q), "ih-extract-multi", exInput[type][q], {"db": {"file": type + str(q) + ".db", "transfer": False}}, arguments, [], dax = self.exdax, walltime = 240)
self._addJob(type + "_extract" + str(q), "ih-extract-multi", exInput[type][q], {"db": {"file": type + str(q) + ".db", "transfer": False}}, arguments, exDep[type][q], walltime = 300)
self._addJob(type + "_extract" + str(q), "ih-extract-multi", exInput[type][q], {"db": {"file": type + str(q) + ".db", "transfer": False}}, arguments, [], dax = self.exdax, walltime = 300)
maprc.write(type + str(q) + ".db" + " file://" + self.basepath + "/output/" + type + str(q) + ".db" + " pool=\"local\"\n")
binDep.append(type + "_extract" + str(q))
aggIn[type + str(q) + ".db"] = {"file": type + str(q) + ".db", "transfer": False}
......@@ -726,8 +727,8 @@ class ImageProcessor(Workflow):
exInput[type][q]["db"] = {"file": type + str(q) + ".db", "transfer": False}
exInput[type][q]["binfile"] = {"file": map[type] + "_hist_bins.json", "transfer": False}
arguments["--bins"] = "binfile"
self._addJob(type + "_extractBins" + str(q), "ih-extract-multi", exInput[type][q], {}, arguments, ["bin_creation"], walltime = 240)
self._addJob(type + "_extractBins" + str(q), "ih-extract-multi", exInput[type][q], {}, arguments, ["bin_creation"], dax = self.exdax, walltime = 240)
self._addJob(type + "_extractBins" + str(q), "ih-extract-multi", exInput[type][q], {}, arguments, ["bin_creation"], walltime = 480)
self._addJob(type + "_extractBins" + str(q), "ih-extract-multi", exInput[type][q], {}, arguments, ["bin_creation"], dax = self.exdax, walltime = 480)
binDep.append(type + "_extractBins" + str(q))
aggIn["db"] = {"file": "img2.db", "transfer": False}
self._addJob("sql_aggregate2", "ih-sql-aggregate", aggIn, {"img3.db": {"file": "img3.db", "transfer": True}}, {"--db": "db", "--output": "img3.db", "--inputs": " ".join([aggIn[x]["file"] for x in aggIn if x != "db"])}, binDep)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment