From 03514cfbd6d5dca18d2df2b37581d02214815c5d Mon Sep 17 00:00:00 2001
From: Emelie Harstad <eharstad@unl.edu>
Date: Fri, 8 Nov 2019 16:43:33 +0000
Subject: [PATCH] Reworked data handling sections
---
content/Anvil/_index.md | 2 +-
content/Data_Storage/_index.md | 10 ++++++++++
.../data_for_unmc_users_only.md | 2 +-
.../data_storage_overview.md} | 14 +++++++-------
.../integrating_box_with_hcc.md | 2 +-
.../preventing_file_loss.md | 10 +++++-----
.../{Handling_Data => Data_Storage}/using_attic.md | 8 ++++----
.../using_nus_gitlab_instance/_index.md | 2 +-
.../setting_up_gitlab_on_hcc_clusters.md | 0
.../using_the_common_file_system.md | 2 +-
content/Data_Transfer/_index.md | 10 ++++++++++
.../connect_to_cb3_irods.md | 2 +-
.../globus_connect/_index.md | 1 +
.../activating_hcc_cluster_endpoints.md | 2 +-
.../globus_connect/creating_globus_groups.md | 0
.../globus_connect/file_sharing.md | 0
.../file_transfers_between_endpoints.md | 6 +++---
..._transfers_to_and_from_personal_workstations.md | 0
.../globus_command_line_interface.md | 0
.../high_speed_data_transfers.md | 0
.../using_rclone_with_hcc.md | 2 +-
content/quickstarts/connecting/_index.md | 2 +-
.../quickstarts/connecting/for_windows_users.md | 2 +-
23 files changed, 50 insertions(+), 29 deletions(-)
create mode 100644 content/Data_Storage/_index.md
rename content/{Handling_Data => Data_Storage}/data_for_unmc_users_only.md (99%)
rename content/{Handling_Data/_index.md => Data_Storage/data_storage_overview.md} (93%)
rename content/{Handling_Data => Data_Storage}/integrating_box_with_hcc.md (99%)
rename content/{Handling_Data => Data_Storage}/preventing_file_loss.md (95%)
rename content/{Handling_Data => Data_Storage}/using_attic.md (92%)
rename content/{Handling_Data => Data_Storage}/using_nus_gitlab_instance/_index.md (99%)
rename content/{Handling_Data => Data_Storage}/using_nus_gitlab_instance/setting_up_gitlab_on_hcc_clusters.md (100%)
rename content/{Handling_Data => Data_Storage}/using_the_common_file_system.md (99%)
create mode 100644 content/Data_Transfer/_index.md
rename content/{Handling_Data => Data_Transfer}/connect_to_cb3_irods.md (99%)
rename content/{Handling_Data => Data_Transfer}/globus_connect/_index.md (99%)
rename content/{Handling_Data => Data_Transfer}/globus_connect/activating_hcc_cluster_endpoints.md (89%)
rename content/{Handling_Data => Data_Transfer}/globus_connect/creating_globus_groups.md (100%)
rename content/{Handling_Data => Data_Transfer}/globus_connect/file_sharing.md (100%)
rename content/{Handling_Data => Data_Transfer}/globus_connect/file_transfers_between_endpoints.md (88%)
rename content/{Handling_Data => Data_Transfer}/globus_connect/file_transfers_to_and_from_personal_workstations.md (100%)
rename content/{Handling_Data => Data_Transfer}/globus_connect/globus_command_line_interface.md (100%)
rename content/{Handling_Data => Data_Transfer}/high_speed_data_transfers.md (100%)
rename content/{Handling_Data => Data_Transfer}/using_rclone_with_hcc.md (99%)
diff --git a/content/Anvil/_index.md b/content/Anvil/_index.md
index 8d54b402..a2b7b3f2 100644
--- a/content/Anvil/_index.md
+++ b/content/Anvil/_index.md
@@ -165,7 +165,7 @@ precious or irreproducible data should not be placed or left on Anvil**.
Transferring files to or from an instance is similar to doing so
with a personal laptop or workstation. To transfer between an
instance and another HCC resource, both SCP and [Globus
- Connect]({{< relref "/Handling_Data/globus_connect" >}}) can be used. For transferring
+ Connect]({{< relref "/Data_Transfer/globus_connect" >}}) can be used. For transferring
between an instance and a laptop/workstation or another instance,
standard file sharing utilities such as Dropbox or Box can be used.
Globus may also be used, with one stipulation. In order to
diff --git a/content/Data_Storage/_index.md b/content/Data_Storage/_index.md
new file mode 100644
index 00000000..6fd2d2aa
--- /dev/null
+++ b/content/Data_Storage/_index.md
@@ -0,0 +1,10 @@
++++
+title = "Data Storage"
+description = "Storing Data on HCC resources."
+weight = "50"
++++
+
+{{% children description="true" %}}
+
+
+
diff --git a/content/Handling_Data/data_for_unmc_users_only.md b/content/Data_Storage/data_for_unmc_users_only.md
similarity index 99%
rename from content/Handling_Data/data_for_unmc_users_only.md
rename to content/Data_Storage/data_for_unmc_users_only.md
index 9fb44bc2..708c9140 100644
--- a/content/Handling_Data/data_for_unmc_users_only.md
+++ b/content/Data_Storage/data_for_unmc_users_only.md
@@ -1,7 +1,7 @@
+++
title = "Data for UNMC Users Only"
description= "Data storage options for UNMC users"
-weight = 50
+weight = 60
+++
{{% panel theme="danger" header="Sensitive and Protected Data" %}} HCC currently has no storage that is suitable for HIPAA or other PID
diff --git a/content/Handling_Data/_index.md b/content/Data_Storage/data_storage_overview.md
similarity index 93%
rename from content/Handling_Data/_index.md
rename to content/Data_Storage/data_storage_overview.md
index 9082985c..c841e4f0 100644
--- a/content/Handling_Data/_index.md
+++ b/content/Data_Storage/data_storage_overview.md
@@ -1,14 +1,14 @@
+++
-title = "Handling Data"
-description = "How to work with and transfer data to/from HCC resources."
-weight = "50"
+title = "Data Storage Overview"
+description = "Overview of Data Storage on HCC resources."
+weight = "10"
+++
{{% panel theme="danger" header="**Sensitive and Protected Data**" %}}HCC currently has *no storage* that is suitable for **HIPAA** or other **PID** data sets. Users are not permitted to store such data on HCC machines.{{% /panel %}}
All HCC machines have three separate areas for every user to store data,
each intended for a different purpose. In addition, we have a transfer
-service that utilizes [Globus Connect]({{< relref "globus_connect" >}}).
+service that utilizes [Globus Connect]({{< relref "/Data_Transfer/globus_connect" >}}).
{{< figure src="/images/35325560.png" height="500" class="img-border">}}
---
@@ -107,18 +107,18 @@ we will gladly discuss alternatives.
Attic is a near line archive available for purchase at HCC. Attic
provides reliable large data storage that is designed to be more
reliable then `/work`, and larger than `/home`. Access to Attic is done
-through [Globus Connect]({{< relref "globus_connect" >}}).
+through [Globus Connect]({{< relref "/Data_Transfer/globus_connect" >}}).
More details on Attic can be found on HCC's
<a href="https://hcc.unl.edu/attic" class="external-link">Attic</a>
website.
---
-### [Globus Connect]({{< relref "globus_connect" >}})
+### [Globus Connect]({{< relref "/Data_Transfer/globus_connect" >}})
For moving large amounts of data into or out of HCC resources, users are
highly encouraged to consider using [Globus
-Connect]({{< relref "globus_connect" >}}).
+Connect]({{< relref "/Data_Transfer/globus_connect" >}}).
---
### Using Box
diff --git a/content/Handling_Data/integrating_box_with_hcc.md b/content/Data_Storage/integrating_box_with_hcc.md
similarity index 99%
rename from content/Handling_Data/integrating_box_with_hcc.md
rename to content/Data_Storage/integrating_box_with_hcc.md
index fa5fd824..982944ec 100644
--- a/content/Handling_Data/integrating_box_with_hcc.md
+++ b/content/Data_Storage/integrating_box_with_hcc.md
@@ -1,7 +1,7 @@
+++
title = "Integrating Box with HCC"
description = "How to integrate Box with HCC"
-weight = 30
+weight = 50
+++
UNL has come to an arrangement
diff --git a/content/Handling_Data/preventing_file_loss.md b/content/Data_Storage/preventing_file_loss.md
similarity index 95%
rename from content/Handling_Data/preventing_file_loss.md
rename to content/Data_Storage/preventing_file_loss.md
index 8d20435f..d264185f 100644
--- a/content/Handling_Data/preventing_file_loss.md
+++ b/content/Data_Storage/preventing_file_loss.md
@@ -1,7 +1,7 @@
+++
title = "Preventing File Loss"
description = "How to prevent file loss on HCC clusters"
-weight = 60
+weight = 40
+++
Each research group is allocated 50TB of storage in `/work` on HCC
@@ -26,7 +26,7 @@ directories to your personal computer. To avoid filling up your personal
hard-drives, consider using an external drive that can easily be placed
in a fireproof safe or at an off-site location for an extra level of
protection. To do this, you can either use [Globus
-Connect]({{< relref "globus_connect" >}}) or an
+Connect]({{< relref "/Data_Transfer/globus_connect" >}}) or an
SCP client, such
as <a href="https://cyberduck.io/" class="external-link">Cyberduck</a> or <a href="https://winscp.net/eng/index.php" class="external-link">WinSCP</a>.
For help setting up an SCP client, check out our [Quick Start
@@ -45,7 +45,7 @@ Benefits:
- Take advantage of our high speed data transfers (10 Gb/s) when using
Globus Connect or [setup your SCP client to use our dedicated high
speed transfer
- servers]({{< relref "high_speed_data_transfers" >}})
+ servers]({{< relref "/Data_Transfer/high_speed_data_transfers" >}})
Limitations:
@@ -100,8 +100,8 @@ Benefits:
layer against file loss.
- No limits on individual or total file sizes.
- High speed data transfers between Attic and the clusters when using
- [Globus Connect]({{< relref "globus_connect" >}}) and [HCC's high-speed data
- servers]({{< relref "high_speed_data_transfers" >}}).
+ [Globus Connect]({{< relref "/Data_Transfer/globus_connect" >}}) and [HCC's high-speed data
+ servers]({{< relref "/Data_Transfer/high_speed_data_transfers" >}}).
Limitations:
diff --git a/content/Handling_Data/using_attic.md b/content/Data_Storage/using_attic.md
similarity index 92%
rename from content/Handling_Data/using_attic.md
rename to content/Data_Storage/using_attic.md
index dc1c89a3..75ca93fb 100644
--- a/content/Handling_Data/using_attic.md
+++ b/content/Data_Storage/using_attic.md
@@ -9,7 +9,7 @@ provides an economical solution called Attic. Attic is a reliable
<a href="https://en.wikipedia.org/wiki/Nearline_storage" class="external-link">near-line data archive</a> storage
system. The files in Attic can be accessed and shared from anywhere
using [Globus
-Connect]({{< relref "globus_connect" >}}),
+Connect]({{< relref "/Data_Transfer/globus_connect" >}}),
with a fast 10Gb/s link. Also, the data in Attic is backed up between
our Lincoln and Omaha facilities to ensure high availability and
disaster tolerance. The data and user activities on Attic are subject to
@@ -34,7 +34,7 @@ cost, please see the
The easiest and fastest way to access Attic is via Globus. You can
transfer files between your computer, our clusters ($HOME, $WORK, and $COMMON on
Crane or Rhino), and Attic. Here is a detailed tutorial on
-how to set up and use [Globus Connect]({{< relref "globus_connect" >}}). For
+how to set up and use [Globus Connect]({{< relref "/Data_Transfer/globus_connect" >}}). For
Attic, use the Globus Endpoint **hcc\#attic**. Your Attic files are
located at `~, `which is a shortcut
for `/attic/<groupname>/<username>`.
@@ -79,7 +79,7 @@ rsync -avz <username>@attic.unl.edu:remote/source/path /local/destination/path
{{% /panel %}}
You can also access your data on Attic using our [high-speed
-transfer servers]({{< relref "high_speed_data_transfers" >}}) if you prefer.
+transfer servers]({{< relref "/Data_Transfer/high_speed_data_transfers" >}}) if you prefer.
Simply use scp or sftp to connect to one of the transfer servers, and
your directory is mounted at `/attic/<groupname>/<username>`.
@@ -88,7 +88,7 @@ your directory is mounted at `/attic/<groupname>/<username>`.
The usage and quota information for your group and the users in the
group are stored in a file named "disk\_usage.txt" in your group's
-directory (`/attic/<groupname>`). You can use either [Globus Connect]({{< relref "globus_connect" >}}) or
+directory (`/attic/<groupname>`). You can use either [Globus Connect]({{< relref "/Data_Transfer/globus_connect" >}}) or
scp to download it. Your usage and expiration is also shown in the web
interface (see below).
diff --git a/content/Handling_Data/using_nus_gitlab_instance/_index.md b/content/Data_Storage/using_nus_gitlab_instance/_index.md
similarity index 99%
rename from content/Handling_Data/using_nus_gitlab_instance/_index.md
rename to content/Data_Storage/using_nus_gitlab_instance/_index.md
index d850551d..0a34ebff 100644
--- a/content/Handling_Data/using_nus_gitlab_instance/_index.md
+++ b/content/Data_Storage/using_nus_gitlab_instance/_index.md
@@ -1,7 +1,7 @@
+++
title = "Using NU's Gitlab instance"
description = "How to use the NU-hosted git.unl.edu"
-weight = "40"
+weight = "70"
+++
[Git](http://git-scm.com) is a free
diff --git a/content/Handling_Data/using_nus_gitlab_instance/setting_up_gitlab_on_hcc_clusters.md b/content/Data_Storage/using_nus_gitlab_instance/setting_up_gitlab_on_hcc_clusters.md
similarity index 100%
rename from content/Handling_Data/using_nus_gitlab_instance/setting_up_gitlab_on_hcc_clusters.md
rename to content/Data_Storage/using_nus_gitlab_instance/setting_up_gitlab_on_hcc_clusters.md
diff --git a/content/Handling_Data/using_the_common_file_system.md b/content/Data_Storage/using_the_common_file_system.md
similarity index 99%
rename from content/Handling_Data/using_the_common_file_system.md
rename to content/Data_Storage/using_the_common_file_system.md
index 5a5ecdb0..1addd067 100644
--- a/content/Handling_Data/using_the_common_file_system.md
+++ b/content/Data_Storage/using_the_common_file_system.md
@@ -1,7 +1,7 @@
+++
title = "Using the /common File System"
description = "How to use HCC's /common file system"
-weight = 70
+weight = 30
+++
### Quick overview:
diff --git a/content/Data_Transfer/_index.md b/content/Data_Transfer/_index.md
new file mode 100644
index 00000000..d772f48c
--- /dev/null
+++ b/content/Data_Transfer/_index.md
@@ -0,0 +1,10 @@
++++
+title = "Data Transfer"
+description = "Transferring Data to and from HCC resources."
+weight = "55"
++++
+
+{{% children description="true" %}}
+
+
+
diff --git a/content/Handling_Data/connect_to_cb3_irods.md b/content/Data_Transfer/connect_to_cb3_irods.md
similarity index 99%
rename from content/Handling_Data/connect_to_cb3_irods.md
rename to content/Data_Transfer/connect_to_cb3_irods.md
index ed03f1ee..c123dd62 100644
--- a/content/Handling_Data/connect_to_cb3_irods.md
+++ b/content/Data_Transfer/connect_to_cb3_irods.md
@@ -1,7 +1,7 @@
+++
title = "Connecting to CB3 iRODS"
description= "How to connect to the CB3 iRODS instance."
-weight = 80
+weight = 40
+++
{{% panel theme="danger" header="Sensitive and Protected Data" %}}
diff --git a/content/Handling_Data/globus_connect/_index.md b/content/Data_Transfer/globus_connect/_index.md
similarity index 99%
rename from content/Handling_Data/globus_connect/_index.md
rename to content/Data_Transfer/globus_connect/_index.md
index ae0585e1..61830c40 100644
--- a/content/Handling_Data/globus_connect/_index.md
+++ b/content/Data_Transfer/globus_connect/_index.md
@@ -1,6 +1,7 @@
+++
title = "Globus Connect"
description = "Globus Connect overview"
+weight = 20
+++
<a href="https://www.globus.org/globus-connect" class="external-link">Globus Connect</a> is
diff --git a/content/Handling_Data/globus_connect/activating_hcc_cluster_endpoints.md b/content/Data_Transfer/globus_connect/activating_hcc_cluster_endpoints.md
similarity index 89%
rename from content/Handling_Data/globus_connect/activating_hcc_cluster_endpoints.md
rename to content/Data_Transfer/globus_connect/activating_hcc_cluster_endpoints.md
index 697916a5..e58f43d9 100644
--- a/content/Handling_Data/globus_connect/activating_hcc_cluster_endpoints.md
+++ b/content/Data_Transfer/globus_connect/activating_hcc_cluster_endpoints.md
@@ -32,7 +32,7 @@ and will not have to be activated again for the next 7 days.
To transfer files between any two HCC clusters, you will need to
activate both endpoints individually.
-Next, learn how to [make file transfers between HCC endpoints]({{< relref "file_transfers_between_endpoints" >}}) or how to [transfer between HCC endpoints and a personal computer]({{< relref "file_transfers_to_and_from_personal_workstations" >}}).
+Next, learn how to [make file transfers between HCC endpoints]({{< relref "/Data_Transfer/globus_connect/file_transfers_between_endpoints" >}}) or how to [transfer between HCC endpoints and a personal computer]({{< relref "/Data_Transfer/globus_connect/file_transfers_to_and_from_personal_workstations" >}}).
---
diff --git a/content/Handling_Data/globus_connect/creating_globus_groups.md b/content/Data_Transfer/globus_connect/creating_globus_groups.md
similarity index 100%
rename from content/Handling_Data/globus_connect/creating_globus_groups.md
rename to content/Data_Transfer/globus_connect/creating_globus_groups.md
diff --git a/content/Handling_Data/globus_connect/file_sharing.md b/content/Data_Transfer/globus_connect/file_sharing.md
similarity index 100%
rename from content/Handling_Data/globus_connect/file_sharing.md
rename to content/Data_Transfer/globus_connect/file_sharing.md
diff --git a/content/Handling_Data/globus_connect/file_transfers_between_endpoints.md b/content/Data_Transfer/globus_connect/file_transfers_between_endpoints.md
similarity index 88%
rename from content/Handling_Data/globus_connect/file_transfers_between_endpoints.md
rename to content/Data_Transfer/globus_connect/file_transfers_between_endpoints.md
index 83ad090c..85cae545 100644
--- a/content/Handling_Data/globus_connect/file_transfers_between_endpoints.md
+++ b/content/Data_Transfer/globus_connect/file_transfers_between_endpoints.md
@@ -5,16 +5,16 @@ weight = 30
+++
To transfer files between HCC clusters, you will first need to
-[activate]({{< relref "activating_hcc_cluster_endpoints" >}}) the
+[activate]({{< relref "/Data_Transfer/globus_connect/activating_hcc_cluster_endpoints" >}}) the
two endpoints you would like to use (the available endpoints
are: `hcc#crane` `hcc#rhino`, and `hcc#attic`). Once
that has been completed, follow the steps below to begin transferring
files. (Note: You can also transfer files between an HCC endpoint and
any other Globus endpoint for which you have authorized access. That
may include a [personal
-endpoint]({{< relref "file_transfers_to_and_from_personal_workstations" >}}),
+endpoint]({{< relref "/Data_Transfer/globus_connect/file_transfers_to_and_from_personal_workstations" >}}),
a [shared
-endpoint]({{< relref "file_sharing" >}}),
+endpoint]({{< relref "/Data_Transfer/globus_connect/file_sharing" >}}),
or an endpoint on another computing resource or cluster. Once the
endpoints have been activated, the file transfer process is generally
the same regardless of the type of endpoints you use. For demonstration
diff --git a/content/Handling_Data/globus_connect/file_transfers_to_and_from_personal_workstations.md b/content/Data_Transfer/globus_connect/file_transfers_to_and_from_personal_workstations.md
similarity index 100%
rename from content/Handling_Data/globus_connect/file_transfers_to_and_from_personal_workstations.md
rename to content/Data_Transfer/globus_connect/file_transfers_to_and_from_personal_workstations.md
diff --git a/content/Handling_Data/globus_connect/globus_command_line_interface.md b/content/Data_Transfer/globus_connect/globus_command_line_interface.md
similarity index 100%
rename from content/Handling_Data/globus_connect/globus_command_line_interface.md
rename to content/Data_Transfer/globus_connect/globus_command_line_interface.md
diff --git a/content/Handling_Data/high_speed_data_transfers.md b/content/Data_Transfer/high_speed_data_transfers.md
similarity index 100%
rename from content/Handling_Data/high_speed_data_transfers.md
rename to content/Data_Transfer/high_speed_data_transfers.md
diff --git a/content/Handling_Data/using_rclone_with_hcc.md b/content/Data_Transfer/using_rclone_with_hcc.md
similarity index 99%
rename from content/Handling_Data/using_rclone_with_hcc.md
rename to content/Data_Transfer/using_rclone_with_hcc.md
index 2069d6b8..02fc1c26 100755
--- a/content/Handling_Data/using_rclone_with_hcc.md
+++ b/content/Data_Transfer/using_rclone_with_hcc.md
@@ -1,7 +1,7 @@
+++
title = "Using Rclone for File Transfer"
description = "How to use Rclone with HCC"
-weight = 9
+weight =30
+++
Rclone is an open source file transfer tool to make transfering files to and from various cloud resources such as Box, Amazon S3, Microsoft OneDrive, and Google Cloud Storage and your local machine a simpler task. Guides on how to set up a variety of resources to transfer to and from can be found at [rclone's webpage](https://rclone.org/).
diff --git a/content/quickstarts/connecting/_index.md b/content/quickstarts/connecting/_index.md
index 5fe56096..35324ea6 100644
--- a/content/quickstarts/connecting/_index.md
+++ b/content/quickstarts/connecting/_index.md
@@ -17,4 +17,4 @@ The management node does as it sounds, it manages the cluster and provides a cen
The worker nodes are what run and process your jobs that are submitted from the schedulers. Through the use of the schedulers, more work can be efficiently done by squeezing in all jobs possible for the resources requested throughout the nodes. They also allow for fair use computing by making sure one user or group is not using the entire cluster at once and allowing others to use the clusters.
</br></br>
#### Central Storage Array:
-The central storage array allows all of the nodes within the cluster to have access to the same files without needing to transfer them around. HCC has three arrays mounted on the clusters with more details [here]({{< ref "/Handling_Data" >}}).
+The central storage array allows all of the nodes within the cluster to have access to the same files without needing to transfer them around. HCC has three arrays mounted on the clusters with more details [here]({{< ref "/Data_Storage/data_storage_overview" >}}).
diff --git a/content/quickstarts/connecting/for_windows_users.md b/content/quickstarts/connecting/for_windows_users.md
index 7db467e8..343206cc 100644
--- a/content/quickstarts/connecting/for_windows_users.md
+++ b/content/quickstarts/connecting/for_windows_users.md
@@ -111,7 +111,7 @@ File Transferring with HCC Supercomputers
-----------------------------------------
{{% notice info%}}
-For best results when transfering data to and from the clusters, refer to [Handling Data]({{< ref "/Handling_Data" >}})
+For best results when transfering data to and from the clusters, refer to [Data Transfer]({{< ref "/Data_Transfer" >}})
{{%/notice%}}
### SCP
--
GitLab