Updated all of the scripts. Made a convenient script for writing the scripts documentation...

doc/source/scripts.rst

@@ -2,6 +2,15 @@ Scripts
 ========
 
 
+chip-create-run
+^^^^^^^^^^^^^^^
+
+.. argparse::
+    :filename: source/scripts/chip-create-run
+    :func: parser
+    :prog: chip-create-run
+
+
 chip-gen
 ^^^^^^^^
 
@@ -10,26 +19,84 @@ chip-gen
     :func: parser
     :prog: chip-gen
 
-chip-create-run
-^^^^^^^^^^^^^^^
+
+chip-job-ccat-format-bed
+^^^^^^^^^^^^^^^^^^^^^^^^
 
 .. argparse::
-    :filename: source/scripts/chip-create-run
+    :filename: source/scripts/chip-job-ccat-format-bed
     :func: parser
-    :prog: chip-create-run
+    :prog: chip-job-ccat-format-bed
+
+
+chip-job-chr-convert
+^^^^^^^^^^^^^^^^^^^^
+
+.. argparse::
+    :filename: source/scripts/chip-job-chr-convert
+    :func: parser
+    :prog: chip-job-chr-convert
+
+
+chip-job-download-encode
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. argparse::
+    :filename: source/scripts/chip-job-download-encode
+    :func: parser
+    :prog: chip-job-download-encode
+
+
+chip-job-download-gridfs
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. argparse::
+    :filename: source/scripts/chip-job-download-gridfs
+    :func: parser
+    :prog: chip-job-download-gridfs
+
+
+chip-job-music
+^^^^^^^^^^^^^^
+
+.. argparse::
+    :filename: source/scripts/chip-job-music
+    :func: parser
+    :prog: chip-job-music
+
+
+chip-job-peakranger-format
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. argparse::
+    :filename: source/scripts/chip-job-peakranger-format
+    :func: parser
+    :prog: chip-job-peakranger-format
+
+
+chip-job-save-result
+^^^^^^^^^^^^^^^^^^^^
+
+.. argparse::
+    :filename: source/scripts/chip-job-save-result
+    :func: parser
+    :prog: chip-job-save-result
+
 
 chip-meta-download
-^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^
 
 .. argparse::
     :filename: source/scripts/chip-meta-download
     :func: parser
     :prog: chip-meta-download
 
+
 chip-meta-import
-^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^
 
 .. argparse::
     :filename: source/scripts/chip-meta-import
     :func: parser
     :prog: chip-meta-import
+

doc/source/scripts/chip-create-run

@@ -6,9 +6,10 @@ import yaml
 import pprint
 
 parser = argparse.ArgumentParser(description="Create a run file from a list of experiment accessions.")
-parser.add_argument("-H", "--host", dest="host", required=True, help="Database host.")
-parser.add_argument("-u", "--username", dest="username", required=True, help="Database user.")
-parser.add_argument("-p", "--password", dest="password", required=True, help="Database user password.")
+parser.add_argument("-H", "--host", dest="host", default="localhost", help="Database host. (default: %(default)s)")
+parser.add_argument("-u", "--username", dest="username", help="Database username (if required).")
+parser.add_argument("-p", "--password", dest="password", help="Database password (if required).")
 parser.add_argument("-n", "--name", dest="name", required=True, help="Name of the run file to create.")
 parser.add_argument("-f", "--file_type", dest="file_type", default="fastq", help="Type of files to extract (fastq or bam).")
+
 parser.add_argument("-a", "--accessions", dest="accessions", nargs="+", required=True, help="List of experiment accessions to load.")

doc/source/scripts/chip-gen

 #!/usr/bin/env python
 from chipathlon.workflow import Workflow
 import argparse
-import yaml
-import pprint
 
 parser = argparse.ArgumentParser(description="Generates a workflow from a run, param & config file.")
-parser.add_argument("-H", "--host", dest="host", required=True, help="Database host.")
-parser.add_argument("-u", "--username", dest="username", required=True, help="Database user.")
-parser.add_argument("-p", "--password", dest="password", required=True, help="Database user password.")
+parser.add_argument("-H", "--host", dest="host", default="localhost", help="Database host. (default: %(default)s)")
+parser.add_argument("-u", "--username", dest="username", help="Database username (if required).")
+parser.add_argument("-p", "--password", dest="password", help="Database password (if required).")
 parser.add_argument("-d", "--dir", dest="dir", required=True, help="Directory name to generate files in.")
 parser.add_argument("--param", dest="param", required=True, help="Path to param file to load.")
 parser.add_argument("--conf", dest="config", required=True, help="Path to config file to load.")
 parser.add_argument("--run", dest="run", required=True, help="Path to run file to load.")
+parser.add_argument("--properties", dest="properties", required=True, help="Path to pegasus properties file.")
+parser.add_argument("--execute-site", dest="execute_site", required=True, default="local", help="Target execute site.  Sites should be defined in configuration.")
+parser.add_argument("--output-site", dest="output_site", required=True, default="local", help="Target output site.  Site should be defined in configuration.")
 
+parser.add_argument("--no-save-db", dest="save_db", default=True, action="store_false", help="Whether or not to save results to the database.  Default: True")
 parser.add_argument("--rewrite", dest="rewrite", default=False, action="store_true", help="If specified, don't load from the database, rewrite files.")
 parser.add_argument("--debug", dest="debug", default=False, action="store_true", help="Print out more information while generating.")

doc/source/scripts/chip-job-ccat-format-bed

+#!/usr/bin/env python
+import argparse
+import os
+
+# The ccat output is *mostly* correct, however simply sorting is not enough
+# as the fourth column labels are not done in correct chromosome order
+# After sorting the output looks like this:
+# chr1    3086860 3087035 ccat_131620  4   0   5.483551    0.577000
+# chr1    3318040 3318245 ccat_131610  4   0   5.483551    0.577000
+# chr1    3372210 3372465 ccat_87299  5   0   6.854439    0.462000
+# When it should look like this:
+# chr1    3086860 3087035 ccat_0  4   0   5.483551    0.577000    -1
+# chr1    3318040 3318245 ccat_1  4   0   5.483551    0.577000    -1
+# chr1    3372210 3372465 ccat_2  5   0   6.854439    0.462000    -1
+
+parser = argparse.ArgumentParser(description = "Format ccat result files.")
+parser.add_argument("--input", "-i", dest="input", required=True, help="Path to input ccat file.")
+parser.add_argument("--output", "-o", dest="output", required=True, help="Output file to write formatted results.")

doc/source/scripts/chip-job-chr-convert

+#!/usr/bin/env python
+
+import pymysql
+import argparse
+import os
+
+parser = argparse.ArgumentParser(description="Replace chromosome locus with number.")
+parser.add_argument("-b", "--bed", dest="bed", required=True, help="Input bed file.")
+parser.add_argument("-o", "--out", dest="out", required=True, help="Output file.")
+parser.add_argument("-c", "--c", dest="chr", default=False, action="store_true", help="If specified convert from name to chr number, otherwise convert from chr number to name.")
+parser.add_argument("-d", "--db", dest="db", default="hg38", help="Database to load conversion from.")
+parser.add_argument("-s", "--server", dest="server", default="genome-mysql.cse.ucsc.edu", help="Location of mysql server.")
+parser.add_argument("-u", "--user", dest="user", default="genome", help="Username for db login.")
+parser.add_argument("-p", "--password", dest="password", default="", help="Password for db login.")
+parser.add_argument("-t", "--table", dest="table", default="ucscToINSDC", help="Table to retrieve locus -> number info from.")

doc/source/scripts/chip-job-download-encode

+#!/usr/bin/env python
+import chipathlon.utils
+import argparse
+
+parser = argparse.ArgumentParser(description="Download target file.")
+parser.add_argument("-u", "--url", dest="url", required=True, help="Target url.")
+parser.add_argument("-p", "--path", dest="path", required=True, help="Local path to file.")
+parser.add_argument("-t", "--url_type", dest="url_type", default="http://", help="Type of url to access.")
+parser.add_argument("-r", "--retries", dest="retries", default=3, type=int, help="Number of retries.")
+parser.add_argument("-n", "--overwrite", dest="overwrite", default=True, action="store_false", help="Dont' overwrite local file if exists.")
+parser.add_argument("-m", "--md5", dest="md5", help="Check md5 value against passed value.")

doc/source/scripts/chip-job-download-gridfs

+#!/usr/bin/env python
+import chipathlon.db
+import argparse
+
+parser = argparse.ArgumentParser(description="Download target file from GridFS.")
+parser.add_argument("-H", "--host", dest="host", default="localhost", help="Database host. (default: %(default)s)")
+parser.add_argument("-u", "--username", dest="username", help="Database username (if required).")
+parser.add_argument("-p", "--password", dest="password", help="Database password (if required).")
+parser.add_argument("-i", "--id", dest="gridfs_id", required=True, help="GridFS ID.")
+parser.add_argument("-d", "--destination", dest="destination", required=True, help="Local path to file destination.")
+parser.add_argument("-r", "--retries", dest="retries", default=3, type=int, help="Number of retries.")
+parser.add_argument("-n", "--no-overwrite", dest="overwrite", default=True, action="store_false", help="Don't overwrite local file if it exists.")
+parser.add_argument("-c", "--checkmd5", dest="checkmd5", action="store_true", help="Check md5 value of downloaded file against database value.")

doc/source/scripts/chip-job-music

+#!/usr/bin/env python
+
+import argparse
+import os
+import subprocess
+import shutil
+import glob
+
+parser = argparse.ArgumentParser(description="Run music peak caller prep & peak calling.")
+parser.add_argument("--prefix", dest="prefix", required=True, help="Unique prefixectory to create")
+parser.add_argument("--controls", dest="controls", required=True, nargs="+", help="Control files to process.")
+parser.add_argument("--signals", dest="signals", required=True, nargs="+", help="Signal files to process.")
+parser.add_argument("--mapp", dest="mapp", required=True, help="Path to mapability profile")
+
+parser.add_argument("--peak_type", dest="peak_type", required=True, default="narrow", choices=["narrow", "punctate", "broad"], help="Peak calling type to use.")
+parser.add_argument("--lmapp", dest="lmapp", default="50")

doc/source/scripts/chip-job-peakranger-format

+#!/usr/bin/env python
+import argparse
+import math
+import sys
+import os
+
+# Peakranger has very different output than expected:
+# chr1    180330013   180330834   ranger_fdrPassed_0_pval_2.00647e-193_fdr_2.79703e-190   2.79703e-190    +
+# chr1    106321435   106322114   ranger_fdrPassed_1_pval_5.1557e-147_fdr_3.59352e-144    3.59352e-144    +
+# chr1    37474619    37475638    ranger_fdrPassed_2_pval_5.45776e-144_fdr_2.53604e-141   2.53604e-141    +
+# There is no calculated score, the pval needs to be stripped from a different
+# column, and the strand is on the end instead of the 6th column
+
+parser = argparse.ArgumentParser(description = "Format peakranger output files.")
+parser.add_argument("--input", "-i", dest="input", required=True, help="Input result file from peakranger.")
+parser.add_argument("--output", "-o", dest="output", required=True, help="Output file to write formatted results.")

doc/source/scripts/chip-job-save-result

+#!/usr/bin/env python
+import chipathlon.db
+import argparse
+import yaml
+import os
+
+parser = argparse.ArgumentParser(description="Insert a bed file into the database.")
+parser.add_argument("-u", "--username", dest="username", help="Database username (if required).")
+parser.add_argument("-p", "--password", dest="password", help="Database password (if required).")
+parser.add_argument("-d", "--host", dest="host", default="localhost", help="Database host. (default: %(default)s)")
+parser.add_argument("-f", "--file", dest="file", required=True, help="Path to result file.")
+parser.add_argument("-m", "--meta", dest="meta", required=True, help="Path to meta yaml file.")

doc/source/scripts/chip-meta-download

@@ -12,3 +12,4 @@ import datetime
 
 parser = argparse.ArgumentParser(description="Download raw JSON for all experiments.")
 parser.add_argument("-o", "--output-dir", dest="outputdir", default=os.getcwd(), help="Output directory.  (default: %(default)s)")
+parser.add_argument("-q", "--quiet", action='store_true', help="Quiet mode.  Do not print progress information. (default: false)")

doc/source/scripts/chip-meta-import

@@ -9,9 +9,10 @@ import os
 import os.path
 
 parser = argparse.ArgumentParser(description="Read per-experiment JSON files and create experiment and samples collections.")
-parser.add_argument("-H", "--host", dest="host", default="hcc-anvil-175-9.unl.edu", help="Database host. (default: %(default)s)")
-parser.add_argument("-u", "--username", dest="username", default="aknecht", required=True, help="Database user. (default: %(default)s)")
-parser.add_argument("-p", "--password", dest="password", required=True, help="Database user password.")
+parser.add_argument("-H", "--host", dest="host", default="localhost", help="Database host. (default: %(default)s)")
+parser.add_argument("-u", "--username", dest="username", help="Database username (if required).")
+parser.add_argument("-p", "--password", dest="password", help="Database password (if required).")
 parser.add_argument("-i", "--input-dir", dest="inputdir", default=os.getcwd(), help="Directory containing per-experiment JSON files.  (default: %(default)s)")
 parser.add_argument("-d", "--drop", dest="drop", default=False, action="store_true", help="Drop data if it exists. (default: %(default)s)")
 parser.add_argument("-s", "--samples", dest="samples", default=False, action="store_true", help="Only recreate the samples collection.")
+parser.add_argument("-q", "--quiet", action='store_true', help="Quiet mode.  Do not print progress information. (default: false)")

doc/source/write_scripts.py

+#!/usr/bin/env python
+import os
+import shutil
+import textwrap
+
+"""
+Helper file to get the auto-generated scripts file up to-date!
+The top of the scripts file should just be:
+
+Scripts
+========
+
+With a section for each individual script like this:
+
+chip-gen
+^^^^^^^^
+
+.. argparse::
+    :filename: source/scripts/chip-gen
+    :func: parser
+    :prog: chip-gen
+
+Additionally, we need copies of the scripts from the root level scripts/
+folder.  But we only want their parsing information!  If the script
+actually does anything it won't work... So... It's gross but I promise
+it's worth it.
+"""
+
+# Find the scripts folder from relative to where we are
+target_path = os.path.abspath(__file__) + "/"
+for _ in range(4):
+    target_path = os.path.dirname(target_path)
+target_path = os.path.join(target_path, "scripts")
+
+with open("scripts.rst", "w") as wh:
+    wh.write("Scripts\n========\n\n")
+    for root, dirs, files in os.walk(target_path):
+        for f in sorted(files):
+            # We only want to write files that have an argument parser in them
+            # We only want to write up to the point just before parse_args()
+            with open(os.path.join(root, f), "r") as rh:
+                lines = rh.readlines()
+                if any(["ArgumentParser" in line for line in lines]):
+                    with open(os.path.join("scripts", f), "w") as new_script:
+                        for line in lines:
+                            if "parse_args()" in line:
+                                break
+                            new_script.write(line)
+                    wh.write(textwrap.dedent("""
+                        %s
+                        %s
+
+                        .. argparse::
+                            :filename: source/scripts/%s
+                            :func: parser
+                            :prog: %s
+
+                        """ % (f, "^" * len(f), f, f)))
+        break