Merge branch '7-add-dependency-check' into 'develop'

Resolve "Add Dependency Check"

See merge request !17

.dev/bootstrap.sh

+#/bin/sh
+cp .dev/hooks/pre-commit .git/hooks/

.dev/hooks/pre-commit

+#!/bin/sh
+exec make update_readme

.gitlab-ci.yml

@@ -33,6 +33,16 @@ Test Sonar Scanner:
     - dockerd
   script: 
     - docker run --rm its-registry.unl.edu/unl-its/docker-ci/static-code-analysis sonar-scanner -v
+    - docker run --rm -v "${PWD}:/work" -w /work its-registry.unl.edu/unl-its/docker-ci/detect-secrets -s
+    - docker run --rm -v "${PWD}:/work" -w /work its-registry.unl.edu/unl-its/docker-ci/dependency-check -v
+
+Test PHP Lint:
+  stage: test
+  tags:
+    - linux, dockerd
+  script: 
+    - t/php-lint/tests.sh
+
 
 Test PHP Lint:
   stage: test

Makefile

@@ -20,7 +20,7 @@ BUILD_ARGS := --build-arg VCS_REF=$(GIT_COMMIT)
 
 .PHONY: all clean test $(DOCKERFILES) static-code-analysis mobile delete_dangling_images security publish
 
-all: php-lint php-unit-test magento2 mobile static-code-analysis
+all: php-lint php-unit-test magento2 mobile static-code-analysis detect-secrets dependency-check
 
 # Image Groups
 ####################
@@ -97,6 +97,18 @@ $(DOCKERFILES):
 	@docker run --rm -i nimmis/label-inspector lint < $(@D)/Dockerfile
 	@docker run --rm -i nimmis/label-inspector validate < $(@D)/Dockerfile
 
+
+# detect-secrets
+# ##############
+detect-secrets: detect-secrets/Dockerfile
+	docker build $(BUILD_ARGS) -t ${REPO_ORG}/detect-secrets detect-secrets/
+
+
+# dependency-check
+# ################
+dependency-check: dependency-check/Dockerfile
+	 docker build $(BUILD_ARGS) -t ${REPO_ORG}/dependency-check dependency-check/
+
 #IMAGES_TO_SCAN = $(shell docker images --format '{{.Repository}}:{{.Tag}}' | grep unl-its )
 #security:
 #	@docker login -u ${TENABLE_IO_ACCESS_KEY} -p ${TENABLE_IO_SECRET_KEY} registry.cloud.tenable.com
@@ -112,6 +124,15 @@ publish:
 		docker push $$image; \
 	done
 
+#Update README.md based on the template ._README.md
+TOOLSET = `make all --dry-run | grep 'docker build' | cut -d ' ' -f 6 | sort -u | awk '{print "- " $$0}'`
+README_HEAD = `cat ./_README.md | sed -e '/%CONTAINERS_LIST%/Q'`
+README_TAIL = `cat ./_README.md | sed -e '1,/%CONTAINERS_LIST%/d'`
+README_FOOTER = `echo "\n\r_Last update: \`date\`_\n"`
+update_readme:
+	@echo "$(README_HEAD)  \n\n$(TOOLSET) \n $(README_TAIL) \n$(README_FOOTER)" > README.md
+	@git add README.md
+
 delete_dangling_images:
 	@echo "Removing dangling images"
 	@docker image prune -f
@@ -119,6 +140,7 @@ help:
 	@echo -e "make [all] \n\t # Creates all the images"
 	@echo -e "make <TARGET> \n\t # Creates an specific image"
 	@echo -e "make publish \n\t # Push the images to the UNL registry"
+	@echo -e "make update_readme \n\t Update the README.md file"
 #	@echo -e "make security \n\t # Send the images to Tenable. Use env variables TENABLE_IO_ACCESS_KEY and TENABLE_IO_SECRET_KEY"
 
 debug:

README.md

 [![pipeline status](https://git.unl.edu/unl-its/docker-ci/badges/develop/pipeline.svg)](https://git.unl.edu/unl-its/docker-ci/commits/develop)
 
 # Docker-ci
-Containers used on CI
 
-## Building Images
-To build these images, clone this repository onto a machine with docker and make installed. Run `make` and all of the images will be built and installed as local docker images.
\ No newline at end of file
+## Containers usefull in CI process
+
+### Building Images
+
+To build these images, clone this repository onto a machine with docker and make installed. Run `make` and all of the images will be built and installed as local docker images.  
+This file is automatically updated after execute `git commit` based on the content of file `_README.md` To enable this feature, execute `.dev/bootstrap.sh` after cloning the repo.
+
+### Available tools    
+
+- its-registry.unl.edu/unl-its/docker-ci/android-build-server:latest
+- its-registry.unl.edu/unl-its/docker-ci/dependency-check
+- its-registry.unl.edu/unl-its/docker-ci/detect-secrets
+- its-registry.unl.edu/unl-its/docker-ci/magento2-unit-test:latest
+- its-registry.unl.edu/unl-its/docker-ci/magento2-xml-lint:latest
+- its-registry.unl.edu/unl-its/docker-ci/php-lint:5.6
+- its-registry.unl.edu/unl-its/docker-ci/php-lint:7.0
+- its-registry.unl.edu/unl-its/docker-ci/php-lint:7.1
+- its-registry.unl.edu/unl-its/docker-ci/php-lint:7.2
+- its-registry.unl.edu/unl-its/docker-ci/php-unit-test:5.6
+- its-registry.unl.edu/unl-its/docker-ci/php-unit-test:7.0
+- its-registry.unl.edu/unl-its/docker-ci/php-unit-test:7.1
+- its-registry.unl.edu/unl-its/docker-ci/php-unit-test:7.2
+- its-registry.unl.edu/unl-its/docker-ci/static-code-analysis:android
+- its-registry.unl.edu/unl-its/docker-ci/static-code-analysis:latest
+- its-registry.unl.edu/unl-its/docker-ci/static-code-analysis:php
+- its-registry.unl.edu/unl-its/docker-ci/static-code-analysis:python 
+    
+
+_Last update: Fri May 31 17:38:34 CDT 2019_

_README.md

+[![pipeline status](https://git.unl.edu/unl-its/docker-ci/badges/develop/pipeline.svg)](https://git.unl.edu/unl-its/docker-ci/commits/develop)
+
+# Docker-ci
+
+## Containers usefull in CI process
+
+### Building Images
+
+To build these images, clone this repository onto a machine with docker and make installed. Run `make` and all of the images will be built and installed as local docker images.  
+This file is automatically updated after execute `git commit` based on the content of file `_README.md` To enable this feature, execute `.dev/bootstrap.sh` after cloning the repo.
+
+### Available tools  
+
+%CONTAINERS_LIST%
+  

dependency-check/Dockerfile

+FROM adoptopenjdk/openjdk8:alpine
+
+ARG BUILD_DATE
+ARG VCS_REF
+ARG VERSION
+LABEL org.label-schema.build-date=$BUILD_DATE \
+      org.label-schema.name="OWASP Dependency Check" \
+      org.label-schema.description="Docker image for dependency checking" \
+      org.label-schema.vcs-ref=$VCS_REF \
+      org.label-schema.vendor="University of Nebraska - Lincoln" \
+      org.label-schema.version="0.1.0" \
+      org.label-schema.schema-version="1.0" \
+      maintainer="J.R. Barreras <rbarrerasmilanes@nebraska.edu>"
+
+#ENV DEPENDENCY_CHECK_VERSION 4.0.2-release
+ENV DEPENDENCY_CHECK_VERSION 5.0.0-M3-release
+
+
+WORKDIR /opt
+
+RUN apk add --no-cache curl jq su-exec && \
+    curl --insecure -o ./dependency-check.zip -L https://dl.bintray.com/jeremy-long/owasp/dependency-check-${DEPENDENCY_CHECK_VERSION}.zip  && \
+    unzip dependency-check.zip  && \
+    rm dependency-check.zip && \
+    /opt/dependency-check/bin/dependency-check.sh --updateonly
+
+
+ENV PATH $PATH:/opt/dependency-check/bin
+
+WORKDIR /work
+
+COPY entrypoint.sh /usr/local/bin/
+RUN chmod +x /usr/local/bin/entrypoint.sh
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
+
+CMD ["/usr/local/bin/entrypoint.sh"]

dependency-check/entrypoint.sh

+#!/bin/sh
+set -e
+
+if [ $1 == '-v' ]; then
+	dependency-check.sh -v
+	exit 0
+fi
+
+dependency-check.sh -n --project ${CI_PROJECT_NAMESPACE} -s ./ -f JSON
+cat dependency-check-report.json | jq '.dependencies | .[] | select (.vulnerabilities) | [.fileName, .filePath, .vulnerabilities]' | sed -e "s/\"\/work\///g"
+RESULT=`cat dependency-check-report.json | jq '.dependencies | .[] | select (.vulnerabilities) | [.fileName, .filePath, .vulnerabilities] | length == 0' | sed -e "s/\"\/work\///g"`
+
+if [ "${RESULT}" = "" ]; then
+	exit 0
+else
+    exit 1
+fi

detect-secrets/Dockerfile

+FROM python:3.7-alpine
+
+ARG BUILD_DATE
+ARG VCS_REF
+ARG VERSION
+LABEL org.label-schema.build-date=$BUILD_DATE \
+      org.label-schema.name="Detects secrets" \
+      org.label-schema.description="Docker image to detect secrets in a code base" \
+      org.label-schema.vcs-ref=$VCS_REF \
+      org.label-schema.vendor="University of Nebraska - Lincoln" \
+      org.label-schema.version="0.1.0" \
+      org.label-schema.schema-version="1.0" \
+      maintainer="J.R. Barreras <barreras@unl.edu>"
+
+WORKDIR /work
+
+RUN apk update && apk upgrade && \
+    apk add --no-cache git jq && \
+	pip install detect-secrets
+
+COPY entrypoint.sh /usr/local/bin/
+RUN chmod +x /usr/local/bin/entrypoint.sh
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
+
+CMD ["/usr/local/bin/detect-secrets"]

detect-secrets/entrypoint.sh

+#!/bin/sh
+set -e
+
+OPT=""
+ENTROPY="4"
+while [ -n "$1" ]; do
+	case "$1" in
+    	-s)
+			OPT="scan"
+        ;;
+		-e)
+			shift
+			ENTROPY="$1"
+			;;
+		*)
+			echo "Invalid parameter(s)"
+			echo "Usage:"
+			echo "docker run --rm -v "${PWD}:/work" -w /work its-registry.unl.edu/unl-its/docker-ci/detect-secrets -s -e 3"
+			echo ""
+			exit 2
+	esac
+	shift
+done
+
+if [ ${OPT} = "scan" ]; then
+	OUTPUT=`/usr/local/bin/detect-secrets scan --base64-limit ${ENTROPY}`
+	echo ${OUTPUT} | jq
+	RESULT=`echo ${OUTPUT} | jq '.results | length == 0'`
+    if [ "${RESULT}" = "true" ]; then
+       exit 0
+   	else
+       exit 1
+    fi
+fi

examples/Detect-Secrets.md

+	# Secrets detection example
+
+## Parameters
+
+| Argument | Description		          |
+| :------- | :--------------------------- |
+| -s       | Scans the current directory  |
+| -e value | Entropy threshold (allowed randomness); Default value 4 |
+
+
+## Detects secrets in the current project
+
+- One 'analysis' stage with one job  
+- Allows the job to fail without impacting the rest of the CI (allow_failure: true)
+
+
+``` yml
+stages:
+  - analysis 
+variables:
+  stage: analysis
+  tags:
+    - docker
+  script:
+    - docker run --rm -v "${PWD}:/work" -w /work its-registry.unl.edu/unl-its/docker-ci/detect-secrets -s -e 4.5
+  allow_failure: true
+``` 
+
+# Inline Allowlisting
+To tell detect-secrets to ignore a particular line of code, simply append an inline pragma: allowlist secret comment. For example:
+
+```python
+API_KEY = "blah-blah-but-actually-not-secret"  # pragma: allowlist secret  
+print('hello world')  
+```
+
+Inline commenting syntax for a multitude of languages is supported:  
+
+| Comment Style	| Language Support |
+| :------------ | :-------------- |
+|#				| e.g. Python, Dockerfile, YAML |
+|//				| e.g. Go, C++, Java		|
+|/* */			| e.g. C, Java	|
+|'				| e.g. Visual Basic .NET |
+| --			| e.g. SQL, Haskell |
+| <!-- --!>		| e.g. XML |
+
+This may be a convenient way for you to allowlist secrets, without having to regenerate the entire baseline again. Furthermore, this makes the allowlisted secrets easily searchable, auditable, and maintainable.
+
+_[source](https://github.com/Yelp/detect-secrets#secrets-baseline)_

examples/dependency-check.md

+# Dependecy check example
+
+## Parameters
+
+| Argument | Description                          |
+| :------- | :--------------------------- |
+| -v       | Prints Dependency Check version and exit  |
+| without parameters | Scans the current directory (must be a git repo) |
+
+
+## Detect outdated dependencies in the current project using [OWASP DependencyCheck](https://www.owasp.org/index.php/OWASP_Dependency_Check)
+
+- One 'analysis' stage with one job  
+- Allows the job to fail without impacting the rest of the CI (allow_failure: true)
+
+
+``` yml
+stages:
+  - analysis 
+variables:
+  stage: analysis
+  tags:
+    - docker
+  script:
+    - docker run --rm -v "${PWD}:/work" -w /work its-registry.unl.edu/unl-its/docker-ci/detect-secrets -s -e 4.5
+    - docker run --rm -v "${PWD}:/work" -w /work its-registry.unl.edu/unl-its/docker-ci/dependency-check
+  allow_failure: true
+``` 
+
+## Scan the current directory (must be a git repo)
+
+``` bash
+docker run -it --rm -v "${PWD}:/work" -w /work -e CI_PROJECT_NAMESPACE=`basename $(git rev-parse --show-toplevel)` its-registry.unl.edu/unl-its/docker-ci/dependency-check
+
+```