diff --git a/.dev/bootstrap.sh b/.dev/bootstrap.sh new file mode 100755 index 0000000000000000000000000000000000000000..bcc0fb686b04c4fb9bb3c52e95313effe9f72c8f --- /dev/null +++ b/.dev/bootstrap.sh @@ -0,0 +1,2 @@ +#/bin/sh +cp .dev/hooks/pre-commit .git/hooks/ diff --git a/.dev/hooks/pre-commit b/.dev/hooks/pre-commit new file mode 100755 index 0000000000000000000000000000000000000000..89ebbe8b54a1c7cdf7b951a0371c7982a744a535 --- /dev/null +++ b/.dev/hooks/pre-commit @@ -0,0 +1,2 @@ +#!/bin/sh +exec make update_readme diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f14d4653ec9309e85fa16ca316052116c257beb0..b0829d5c2081fd4f1fa2d20604499cc61e68f5d6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,6 +33,7 @@ test: - dockerd script: - docker run --rm its-registry.unl.edu/unl-its/docker-ci/static-code-analysis sonar-scanner -v + - docker run --rm -v "${PWD}:/work" -w /work its-registry.unl.edu/unl-its/docker-ci/detect-secrets -s Push to Container Registry: after_script: diff --git a/Makefile b/Makefile index eea00f842ff2a4a663b0c1d87e550382513edc37..e73972a6bf079106a10f07c08a4ecef24d6c2c0f 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ BUILD_ARGS := --build-arg VCS_REF=$(GIT_COMMIT) .PHONY: all clean test $(DOCKERFILES) static-code-analysis mobile delete_dangling_images security publish -all: php-lint php-unit-test magento2 mobile static-code-analysis +all: php-lint php-unit-test magento2 mobile static-code-analysis detect-secrets # Image Groups #################### @@ -97,6 +97,13 @@ $(DOCKERFILES): @docker run --rm -i nimmis/label-inspector lint < $(@D)/Dockerfile @docker run --rm -i nimmis/label-inspector validate < $(@D)/Dockerfile + +# detect-secrets +# ############## +detect-secrets: detect-secrets/Dockerfile + docker build $(BUILD_ARGS) -t ${REPO_ORG}/detect-secrets detect-secrets/ + + #IMAGES_TO_SCAN = $(shell docker images --format '{{.Repository}}:{{.Tag}}' | grep unl-its ) #security: # @docker login -u ${TENABLE_IO_ACCESS_KEY} -p ${TENABLE_IO_SECRET_KEY} registry.cloud.tenable.com @@ -112,6 +119,15 @@ publish: docker push $$image; \ done +#Update README.md based on the template ._README.md +TOOLSET := `echo "$(sort $(IMAGES_TO_PUBLISH))" | sed -e "s/ /\n - /g"))` +README_HEAD = `cat ./_README.md | sed -e '/%CONTAINERS_LIST%/Q'` +README_TAIL = `cat ./_README.md | sed -e '1,/%CONTAINERS_LIST%/d'` +README_FOOTER = `echo "\n\r_Last update: \`date\`_\n"` +update_readme: + @echo "$(README_HEAD) \n\n - $(TOOLSET) \n $(README_TAIL) \n$(README_FOOTER)" > README.md + @git add README.md + delete_dangling_images: @echo "Removing dangling images" @docker image prune -f @@ -119,6 +135,7 @@ help: @echo -e "make [all] \n\t # Creates all the images" @echo -e "make <TARGET> \n\t # Creates an specific image" @echo -e "make publish \n\t # Push the images to the UNL registry" + @echo -e "make update_readme \n\t Update the README.md file" # @echo -e "make security \n\t # Send the images to Tenable. Use env variables TENABLE_IO_ACCESS_KEY and TENABLE_IO_SECRET_KEY" debug: diff --git a/README.md b/README.md index 7c1f31e244d211b5e709ea4cf67c8a38ddc36e3d..0b470a5ecb583411a23ee004c3deddafb6c75181 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,30 @@ [](https://git.unl.edu/unl-its/docker-ci/commits/develop) # Docker-ci -Containers used on CI -## Building Images -To build these images, clone this repository onto a machine with docker and make installed. Run `make` and all of the images will be built and installed as local docker images. \ No newline at end of file +## Containers usefull in CI process + +### Building Images + +To build these images, clone this repository onto a machine with docker and make installed. Run `make` and all of the images will be built and installed as local docker images. +This file is automatically updated after execute `git commit` based on the content of file `_README.md` To enable this feature, execute `.dev/bootstrap.sh` after cloning the repo. + +### Available tools + + - its-registry.unl.edu/unl-its/docker-ci/detect-secrets:latest + - its-registry.unl.edu/unl-its/docker-ci/magento2-unit-test:latest + - its-registry.unl.edu/unl-its/docker-ci/magento2-xml-lint:latest + - its-registry.unl.edu/unl-its/docker-ci/php-lint:5.6 + - its-registry.unl.edu/unl-its/docker-ci/php-lint:7.0 + - its-registry.unl.edu/unl-its/docker-ci/php-lint:7.1 + - its-registry.unl.edu/unl-its/docker-ci/php-lint:7.2 + - its-registry.unl.edu/unl-its/docker-ci/php-lint:latest + - its-registry.unl.edu/unl-its/docker-ci/php-unit-test:5.6 + - its-registry.unl.edu/unl-its/docker-ci/php-unit-test:7.1 + - its-registry.unl.edu/unl-its/docker-ci/secrets-detection:latest + - its-registry.unl.edu/unl-its/docker-ci/static-code-analysis:latest + - its-registry.unl.edu/unl-its/docker-ci/static-code-analysis:php + - its-registry.unl.edu/unl-its/docker-ci/static-code-analysis:python + + + _Last update: Fri May 31 12:11:19 CDT 2019_ diff --git a/_README.md b/_README.md new file mode 100644 index 0000000000000000000000000000000000000000..9f6b66da850c378747d3a896d1d11259f5a5f960 --- /dev/null +++ b/_README.md @@ -0,0 +1,15 @@ +[](https://git.unl.edu/unl-its/docker-ci/commits/develop) + +# Docker-ci + +## Containers usefull in CI process + +### Building Images + +To build these images, clone this repository onto a machine with docker and make installed. Run `make` and all of the images will be built and installed as local docker images. +This file is automatically updated after execute `git commit` based on the content of file `_README.md` To enable this feature, execute `.dev/bootstrap.sh` after cloning the repo. + +### Available tools + +%CONTAINERS_LIST% + diff --git a/detect-secrets/Dockerfile b/detect-secrets/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..390f8c3581853da6ee43bb8aedfb4b1e2657a559 --- /dev/null +++ b/detect-secrets/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.7-alpine + +ARG BUILD_DATE +ARG VCS_REF +ARG VERSION +LABEL org.label-schema.build-date=$BUILD_DATE \ + org.label-schema.name="Detects secrets" \ + org.label-schema.description="Docker image to detect secrets in a code base" \ + org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.vendor="University of Nebraska - Lincoln" \ + org.label-schema.version="0.1.0" \ + org.label-schema.schema-version="1.0" \ + maintainer="J.R. Barreras <barreras@unl.edu>" + +WORKDIR /work + +RUN apk update && apk upgrade && \ + apk add --no-cache git jq && \ + pip install detect-secrets + +COPY entrypoint.sh /usr/local/bin/ +RUN chmod +x /usr/local/bin/entrypoint.sh +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] + +CMD ["/usr/local/bin/detect-secrets"] diff --git a/detect-secrets/entrypoint.sh b/detect-secrets/entrypoint.sh new file mode 100644 index 0000000000000000000000000000000000000000..0bbe5c0a73f1818f27daf14df44645754d3f4a2f --- /dev/null +++ b/detect-secrets/entrypoint.sh @@ -0,0 +1,34 @@ +#!/bin/sh +set -e + +OPT="" +ENTROPY="4" +while [ -n "$1" ]; do + case "$1" in + -s) + OPT="scan" + ;; + -e) + shift + ENTROPY="$1" + ;; + *) + echo "Invalid parameter(s)" + echo "Usage:" + echo "docker run --rm -v "${PWD}:/work" -w /work its-registry.unl.edu/unl-its/docker-ci/detect-secrets -s -e 3" + echo "" + exit 2 + esac + shift +done + +if [ ${OPT} = "scan" ]; then + OUTPUT=`/usr/local/bin/detect-secrets scan --base64-limit ${ENTROPY}` + echo ${OUTPUT} | jq + RESULT=`echo ${OUTPUT} | jq '.results | length == 0'` + if [ "${RESULT}" = "true" ]; then + exit 0 + else + exit 1 + fi +fi diff --git a/examples/Detect-Secrets.md b/examples/Detect-Secrets.md new file mode 100644 index 0000000000000000000000000000000000000000..49ba574fb07b36f22047029a47a8119f688c737b --- /dev/null +++ b/examples/Detect-Secrets.md @@ -0,0 +1,50 @@ + # Secrets detection example + +## Parameters + +| Argument | Description | +| :------- | :--------------------------- | +| -s | Scans the current directory | +| -e value | Entropy threshold (allowed randomness); Default value 4 | + + +## Detects secrets in the current project + +- One 'analysis' stage with one job +- Allows the job to fail without impacting the rest of the CI (allow_failure: true) + + +``` yml +stages: + - analysis +variables: + stage: analysis + tags: + - docker + script: + - docker run --rm -v "${PWD}:/work" -w /work its-registry.unl.edu/unl-its/docker-ci/detect-secrets -s -e 4.5 + allow_failure: true +``` + +# Inline Allowlisting +To tell detect-secrets to ignore a particular line of code, simply append an inline pragma: allowlist secret comment. For example: + +```python +API_KEY = "blah-blah-but-actually-not-secret" # pragma: allowlist secret +print('hello world') +``` + +Inline commenting syntax for a multitude of languages is supported: + +| Comment Style | Language Support | +| :------------ | :-------------- | +|# | e.g. Python, Dockerfile, YAML | +|// | e.g. Go, C++, Java | +|/* */ | e.g. C, Java | +|' | e.g. Visual Basic .NET | +| -- | e.g. SQL, Haskell | +| <!-- --!> | e.g. XML | + +This may be a convenient way for you to allowlist secrets, without having to regenerate the entire baseline again. Furthermore, this makes the allowlisted secrets easily searchable, auditable, and maintainable. + +_[source](https://github.com/Yelp/detect-secrets#secrets-baseline)_