Merge branch '6-add-automated-secrets-detection' into '7-add-dependency-check'

sync dependency-check with secrets-detection

See merge request !18

.dev/bootstrap.sh

+#/bin/sh
+cp .dev/hooks/pre-commit .git/hooks/

.dev/hooks/pre-commit

+#!/bin/sh
+exec make update_readme

.gitlab-ci.yml

@@ -33,6 +33,7 @@ test:
     - dockerd
   script: 
     - docker run --rm its-registry.unl.edu/unl-its/docker-ci/static-code-analysis sonar-scanner -v
+    - docker run --rm -v "${PWD}:/work" -w /work its-registry.unl.edu/unl-its/docker-ci/detect-secrets -s
 
 Push to Container Registry:
   after_script:

Makefile

@@ -20,7 +20,7 @@ BUILD_ARGS := --build-arg VCS_REF=$(GIT_COMMIT)
 
 .PHONY: all clean test $(DOCKERFILES) static-code-analysis mobile delete_dangling_images security publish
 
-all: php-lint php-unit-test magento2 mobile static-code-analysis
+all: php-lint php-unit-test magento2 mobile static-code-analysis detect-secrets
 
 # Image Groups
 ####################
@@ -97,6 +97,13 @@ $(DOCKERFILES):
 	@docker run --rm -i nimmis/label-inspector lint < $(@D)/Dockerfile
 	@docker run --rm -i nimmis/label-inspector validate < $(@D)/Dockerfile
 
+
+# detect-secrets
+# ##############
+detect-secrets: detect-secrets/Dockerfile
+	docker build $(BUILD_ARGS) -t ${REPO_ORG}/detect-secrets detect-secrets/
+
+
 #IMAGES_TO_SCAN = $(shell docker images --format '{{.Repository}}:{{.Tag}}' | grep unl-its )
 #security:
 #	@docker login -u ${TENABLE_IO_ACCESS_KEY} -p ${TENABLE_IO_SECRET_KEY} registry.cloud.tenable.com
@@ -112,6 +119,15 @@ publish:
 		docker push $$image; \
 	done
 
+#Update README.md based on the template ._README.md
+TOOLSET := `echo "$(sort $(IMAGES_TO_PUBLISH))" | sed -e "s/ /\n - /g"))`
+README_HEAD = `cat ./_README.md | sed -e '/%CONTAINERS_LIST%/Q'`
+README_TAIL = `cat ./_README.md | sed -e '1,/%CONTAINERS_LIST%/d'`
+README_FOOTER = `echo "\n\r_Last update: \`date\`_\n"`
+update_readme:
+	@echo "$(README_HEAD)  \n\n - $(TOOLSET) \n $(README_TAIL) \n$(README_FOOTER)" > README.md
+	@git add README.md
+
 delete_dangling_images:
 	@echo "Removing dangling images"
 	@docker image prune -f
@@ -119,6 +135,7 @@ help:
 	@echo -e "make [all] \n\t # Creates all the images"
 	@echo -e "make <TARGET> \n\t # Creates an specific image"
 	@echo -e "make publish \n\t # Push the images to the UNL registry"
+	@echo -e "make update_readme \n\t Update the README.md file"
 #	@echo -e "make security \n\t # Send the images to Tenable. Use env variables TENABLE_IO_ACCESS_KEY and TENABLE_IO_SECRET_KEY"
 
 debug:

README.md

 [![pipeline status](https://git.unl.edu/unl-its/docker-ci/badges/develop/pipeline.svg)](https://git.unl.edu/unl-its/docker-ci/commits/develop)
 
 # Docker-ci
-Containers used on CI
 
-## Building Images
+## Containers usefull in CI process
+
+### Building Images
+
 To build these images, clone this repository onto a machine with docker and make installed. Run `make` and all of the images will be built and installed as local docker images.  
+This file is automatically updated after execute `git commit` based on the content of file `_README.md` To enable this feature, execute `.dev/bootstrap.sh` after cloning the repo.
+
+### Available tools    
+
+ - its-registry.unl.edu/unl-its/docker-ci/detect-secrets:latest
+ - its-registry.unl.edu/unl-its/docker-ci/magento2-unit-test:latest
+ - its-registry.unl.edu/unl-its/docker-ci/magento2-xml-lint:latest
+ - its-registry.unl.edu/unl-its/docker-ci/php-lint:5.6
+ - its-registry.unl.edu/unl-its/docker-ci/php-lint:7.0
+ - its-registry.unl.edu/unl-its/docker-ci/php-lint:7.1
+ - its-registry.unl.edu/unl-its/docker-ci/php-lint:7.2
+ - its-registry.unl.edu/unl-its/docker-ci/php-lint:latest
+ - its-registry.unl.edu/unl-its/docker-ci/php-unit-test:5.6
+ - its-registry.unl.edu/unl-its/docker-ci/php-unit-test:7.1
+ - its-registry.unl.edu/unl-its/docker-ci/secrets-detection:latest
+ - its-registry.unl.edu/unl-its/docker-ci/static-code-analysis:latest
+ - its-registry.unl.edu/unl-its/docker-ci/static-code-analysis:php
+ - its-registry.unl.edu/unl-its/docker-ci/static-code-analysis:python 
+    
+
+_Last update: Fri May 31 12:11:19 CDT 2019_

_README.md

+[![pipeline status](https://git.unl.edu/unl-its/docker-ci/badges/develop/pipeline.svg)](https://git.unl.edu/unl-its/docker-ci/commits/develop)
+
+# Docker-ci
+
+## Containers usefull in CI process
+
+### Building Images
+
+To build these images, clone this repository onto a machine with docker and make installed. Run `make` and all of the images will be built and installed as local docker images.  
+This file is automatically updated after execute `git commit` based on the content of file `_README.md` To enable this feature, execute `.dev/bootstrap.sh` after cloning the repo.
+
+### Available tools  
+
+%CONTAINERS_LIST%
+  

detect-secrets/Dockerfile

+FROM python:3.7-alpine
+
+ARG BUILD_DATE
+ARG VCS_REF
+ARG VERSION
+LABEL org.label-schema.build-date=$BUILD_DATE \
+      org.label-schema.name="Detects secrets" \
+      org.label-schema.description="Docker image to detect secrets in a code base" \
+      org.label-schema.vcs-ref=$VCS_REF \
+      org.label-schema.vendor="University of Nebraska - Lincoln" \
+      org.label-schema.version="0.1.0" \
+      org.label-schema.schema-version="1.0" \
+      maintainer="J.R. Barreras <barreras@unl.edu>"
+
+WORKDIR /work
+
+RUN apk update && apk upgrade && \
+    apk add --no-cache git jq && \
+	pip install detect-secrets
+
+COPY entrypoint.sh /usr/local/bin/
+RUN chmod +x /usr/local/bin/entrypoint.sh
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
+
+CMD ["/usr/local/bin/detect-secrets"]

detect-secrets/entrypoint.sh

+#!/bin/sh
+set -e
+
+OPT=""
+ENTROPY="4"
+while [ -n "$1" ]; do
+	case "$1" in
+    	-s)
+			OPT="scan"
+        ;;
+		-e)
+			shift
+			ENTROPY="$1"
+			;;
+		*)
+			echo "Invalid parameter(s)"
+			echo "Usage:"
+			echo "docker run --rm -v "${PWD}:/work" -w /work its-registry.unl.edu/unl-its/docker-ci/detect-secrets -s -e 3"
+			echo ""
+			exit 2
+	esac
+	shift
+done
+
+if [ ${OPT} = "scan" ]; then
+	OUTPUT=`/usr/local/bin/detect-secrets scan --base64-limit ${ENTROPY}`
+	echo ${OUTPUT} | jq
+	RESULT=`echo ${OUTPUT} | jq '.results | length == 0'`
+    if [ "${RESULT}" = "true" ]; then
+       exit 0
+   	else
+       exit 1
+    fi
+fi

examples/Detect-Secrets.md

+	# Secrets detection example
+
+## Parameters
+
+| Argument | Description		          |
+| :------- | :--------------------------- |
+| -s       | Scans the current directory  |
+| -e value | Entropy threshold (allowed randomness); Default value 4 |
+
+
+## Detects secrets in the current project
+
+- One 'analysis' stage with one job  
+- Allows the job to fail without impacting the rest of the CI (allow_failure: true)
+
+
+``` yml
+stages:
+  - analysis 
+variables:
+  stage: analysis
+  tags:
+    - docker
+  script:
+    - docker run --rm -v "${PWD}:/work" -w /work its-registry.unl.edu/unl-its/docker-ci/detect-secrets -s -e 4.5
+  allow_failure: true
+``` 
+
+# Inline Allowlisting
+To tell detect-secrets to ignore a particular line of code, simply append an inline pragma: allowlist secret comment. For example:
+
+```python
+API_KEY = "blah-blah-but-actually-not-secret"  # pragma: allowlist secret  
+print('hello world')  
+```
+
+Inline commenting syntax for a multitude of languages is supported:  
+
+| Comment Style	| Language Support |
+| :------------ | :-------------- |
+|#				| e.g. Python, Dockerfile, YAML |
+|//				| e.g. Go, C++, Java		|
+|/* */			| e.g. C, Java	|
+|'				| e.g. Visual Basic .NET |
+| --			| e.g. SQL, Haskell |
+| <!-- --!>		| e.g. XML |
+
+This may be a convenient way for you to allowlist secrets, without having to regenerate the entire baseline again. Furthermore, this makes the allowlisted secrets easily searchable, auditable, and maintainable.
+
+_[source](https://github.com/Yelp/detect-secrets#secrets-baseline)_