Skip to content
GitLab
Commit 03badd2f authored by aknecht2's avatar aknecht2
Browse files

Updated workflow parsing to require a properties file. Executable loading is... 

Updated workflow parsing to require a properties file.  Executable loading is now done based on the provided chipathlon_bin path.
parent d4bfd6f9
Hide whitespace changes
Inline Side-by-side
chipathlon/conf.py
View file @ 03badd2f
......@@ -167,3 +167,12 @@ genomes = {
"additional_files": file_extensions["bowtie2_genome"]
}
}
config_file = {
"required_keys": [
"chipathlon_bin", "idr_bin", "pegasus_home", "email"
],
"optional_keys": [
"arch", "os"
]
}
chipathlon/pegasus_config.py deleted 100644 → 0
View file @ d4bfd6f9
import yaml
import os
import xml.dom.minidom
class PegasusConfig(object):
"""
:param config_file: Path to configuration yaml file used for pegasus.
:type config_file: str
:param base_path: Base path of workflow generation.
:type base_path: str
The PegasusConfig loads all information from the config file to create
the proper sites.xml and conf.rc pegasus needs to run. Additionally,
the PegasusConfig class holds information about relevant sites.
"""
def __init__(self, config_file, base_path):
self.config_file = config_file
self.base_path = base_path
self.errors = []
self.site_file = "%s/input/sites.xml" % (base_path,)
self.properties_file = "%s/input/properties.conf" % (base_path,)
try:
with open(config_file, "r") as rh:
self.config = yaml.load(rh)
self.validate()
except yaml.YAMLError as exc:
self.errors.append("Error parsing config template file '%s': %s.\n" % (config_file, exc))
return
def __getitem__(self, key):
return self.config[key]
def is_valid(self):
"""
:returns: Whether or not the run is valid.
"""
return len(self.errors) == 0
def get_error_string(self):
"""
:returns: The errors as a newline separated string.
"""
return "\n".join(self.errors)
def validate(self):
"""
Validates the passed in configuration file. Many values passed in
cannot be validated before run-time however. Be careful.
"""
self._validate_notify()
self._validate_sites()
if "install_paths" in self.config:
if len(self.config["install_paths"]) < 0:
self.errors.append("Error parsing config template file '%s': At least 1 install_path needs to be specified." % (self.config_file,))
else:
self.errors.append("Error parsing config template file '%s': install_paths must be included." % (self.config_file,))
return
def _validate_sites(self):
if "sites" in self.config:
if len(self.config["sites"].keys()) > 0:
for site in self.config["sites"]:
if "dirs" in self.config["sites"][site]:
if len(self.config["sites"][site]["dirs"].keys()) == 0:
self.errors.append("Error parsing config template file '%s': Site '%s' has 0 dirs defined." % (self.config_file, site))
else:
self.errors.append("Error parsing config template file '%s': Site '%s' does not have any dirs defined." % (self.config_file, site))
else:
self.errors.append("Error parsing config template file '%s': At least 1 site must be defined." % (self.config_file,))
else:
self.errors.append("Error parsing config template file '%s': sites must be defined in config." % (self.config_file,))
return
def _validate_notify(self):
if "notify" in self.config:
if "pegasus_home" in self.config["notify"]:
if not os.path.isdir(self.config["notify"]["pegasus_home"]):
self.errors.append("Error parsing config template file '%s': pegasus_home path '%s' does not exist." % (config_file, self.config["notify"]["pegasus_home"]))
else:
self.errors.append("Error parsing config template file '%s': pegasus_home must be defined under 'notify'." % (config_file,))
if not "email" in self.config["notify"]:
self.errors.append("Error parsing config template file '%s': email must be defined under 'notify'." % (config_file,))
else:
self.errors.append("Error parsing config template file '%s': notify must be defined at root level of config." % (config_file,))
return
def create_sites(self):
"""
Creates the correct sites.xml file from the input configuration data.
"""
site_string = """<sitecatalog xmlns="http://pegasus.isi.edu/schema/sitecatalog" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://pegasus.isi.edu/schema/sitecatalog http://pegasus.isi.edu/schema/sc-4.0.xsd" version="4.0">\n"""
for site_name, site_info in self.config["sites"].iteritems():
# Load initial site info
site_string += """<site handle="%s" arch="%s" os="%s">\n""" % (
site_name,
site_info.get("arch", "x86_64"),
site_info.get("os", "LINUX")
)
# Load each directory for the site
for dir_type, dir_path in site_info["dirs"].iteritems():
site_string += """\
<directory type="%s" path="%s">
<file-server operation="all" url="file://%s" />
</directory>
""" % (dir_type, dir_path.replace("%base_path%", self.base_path), dir_path.replace("%base_path%", self.base_path))
# Grids is required for the BRIDGES setup,
# conditionally add those to the sites
if "grids" in site_info:
site_string += "\n"
for grid in site_info["grids"]:
site_string += """<grid type="%s" contact="%s" scheduler="%s" jobtype="%s" />\n""" % (
grid.get("type", "batch"),
grid.get("contact"),
grid.get("scheduler", "slurm"),
grid.get("jobtype")
)
# Pass profile information directly through
if "profile" in site_info:
site_string += "\n"
for namespace in site_info["profile"]:
for key, value in site_info["profile"][namespace].iteritems():
if key == "PATH":
value = value + ":%s/jobs/scripts/" % (os.path.dirname(os.path.abspath(__file__)),)
site_string += """\n\t<profile namespace="%s" key="%s">%s</profile> """ % (
namespace,
key,
value
)
# If PATH is not specified, we need to add a key for it
if "env" not in site_info["profile"] and "PATH" not in site_info["profile"]["env"]:
site_string += """\n\t<profile namespace="env" key="PATH">%s/jobs/scripts/</profile> """ % (
os.path.dirname(os.path.abspath(__file__))
)
site_string += "</site>\n"
site_string += "</sitecatalog>\n"
with open(self.site_file, "w") as wh:
wh.write("\n".join([line for line in xml.dom.minidom.parseString(site_string).toprettyxml().split("\n") if line.strip()]))
return
def create_properties(self):
"""
Creates the correct properties.conf file from the input configuration data.
"""
properties_string = """\
pegasus.catalog.site = XML
pegasus.catalog.site.file = %s
pegasus.dir.useTimestamp = true
pegasus.condor.logs.symlink = false
pegasus.data.configuration = sharedfs
pegasus.transfer.links = true
""" % (self.site_file,)
if "properties" in self.config:
for key, value in self.config["properties"].iteritems():
properties_string += "%s = %s\n" % (key, value)
with open(self.properties_file, "w") as wh:
wh.write(properties_string)
return
chipathlon/workflow.py
View file @ 03badd2f
No preview for this file type
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment