diff --git a/.dev/bootstrap.sh b/.dev/bootstrap.sh new file mode 100755 index 0000000000000000000000000000000000000000..bcc0fb686b04c4fb9bb3c52e95313effe9f72c8f --- /dev/null +++ b/.dev/bootstrap.sh @@ -0,0 +1,2 @@ +#/bin/sh +cp .dev/hooks/pre-commit .git/hooks/ diff --git a/.dev/hooks/pre-commit b/.dev/hooks/pre-commit new file mode 100755 index 0000000000000000000000000000000000000000..89ebbe8b54a1c7cdf7b951a0371c7982a744a535 --- /dev/null +++ b/.dev/hooks/pre-commit @@ -0,0 +1,2 @@ +#!/bin/sh +exec make update_readme diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f9dd67138a230d04d42845c9ebd59f8e0e300c3b..ad95c7d951bd9cb162cd8ce78b4d9854c19cedd9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,6 +33,16 @@ Test Sonar Scanner: - dockerd script: - docker run --rm its-registry.unl.edu/unl-its/docker-ci/static-code-analysis sonar-scanner -v + - docker run --rm -v "${PWD}:/work" -w /work its-registry.unl.edu/unl-its/docker-ci/detect-secrets -s + - docker run --rm -v "${PWD}:/work" -w /work its-registry.unl.edu/unl-its/docker-ci/dependency-check -v + +Test PHP Lint: + stage: test + tags: + - linux, dockerd + script: + - t/php-lint/tests.sh + Test PHP Lint: stage: test diff --git a/Makefile b/Makefile index eea00f842ff2a4a663b0c1d87e550382513edc37..155e4b068ab1cfd80a24e349a0964d34c603682c 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ BUILD_ARGS := --build-arg VCS_REF=$(GIT_COMMIT) .PHONY: all clean test $(DOCKERFILES) static-code-analysis mobile delete_dangling_images security publish -all: php-lint php-unit-test magento2 mobile static-code-analysis +all: php-lint php-unit-test magento2 mobile static-code-analysis detect-secrets dependency-check # Image Groups #################### @@ -97,6 +97,18 @@ $(DOCKERFILES): @docker run --rm -i nimmis/label-inspector lint < $(@D)/Dockerfile @docker run --rm -i nimmis/label-inspector validate < $(@D)/Dockerfile + +# detect-secrets +# ############## +detect-secrets: detect-secrets/Dockerfile + docker build $(BUILD_ARGS) -t ${REPO_ORG}/detect-secrets detect-secrets/ + + +# dependency-check +# ################ +dependency-check: dependency-check/Dockerfile + docker build $(BUILD_ARGS) -t ${REPO_ORG}/dependency-check dependency-check/ + #IMAGES_TO_SCAN = $(shell docker images --format '{{.Repository}}:{{.Tag}}' | grep unl-its ) #security: # @docker login -u ${TENABLE_IO_ACCESS_KEY} -p ${TENABLE_IO_SECRET_KEY} registry.cloud.tenable.com @@ -112,6 +124,15 @@ publish: docker push $$image; \ done +#Update README.md based on the template ._README.md +TOOLSET = `make all --dry-run | grep 'docker build' | cut -d ' ' -f 6 | sort -u | awk '{print "- " $$0}'` +README_HEAD = `cat ./_README.md | sed -e '/%CONTAINERS_LIST%/Q'` +README_TAIL = `cat ./_README.md | sed -e '1,/%CONTAINERS_LIST%/d'` +README_FOOTER = `echo "\n\r_Last update: \`date\`_\n"` +update_readme: + @echo "$(README_HEAD) \n\n$(TOOLSET) \n $(README_TAIL) \n$(README_FOOTER)" > README.md + @git add README.md + delete_dangling_images: @echo "Removing dangling images" @docker image prune -f @@ -119,6 +140,7 @@ help: @echo -e "make [all] \n\t # Creates all the images" @echo -e "make <TARGET> \n\t # Creates an specific image" @echo -e "make publish \n\t # Push the images to the UNL registry" + @echo -e "make update_readme \n\t Update the README.md file" # @echo -e "make security \n\t # Send the images to Tenable. Use env variables TENABLE_IO_ACCESS_KEY and TENABLE_IO_SECRET_KEY" debug: diff --git a/README.md b/README.md index 7c1f31e244d211b5e709ea4cf67c8a38ddc36e3d..1ad9cb945b282f6f50c56932c9705898f5c61434 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,33 @@ [](https://git.unl.edu/unl-its/docker-ci/commits/develop) # Docker-ci -Containers used on CI -## Building Images -To build these images, clone this repository onto a machine with docker and make installed. Run `make` and all of the images will be built and installed as local docker images. \ No newline at end of file +## Containers usefull in CI process + +### Building Images + +To build these images, clone this repository onto a machine with docker and make installed. Run `make` and all of the images will be built and installed as local docker images. +This file is automatically updated after execute `git commit` based on the content of file `_README.md` To enable this feature, execute `.dev/bootstrap.sh` after cloning the repo. + +### Available tools + +- its-registry.unl.edu/unl-its/docker-ci/android-build-server:latest +- its-registry.unl.edu/unl-its/docker-ci/dependency-check +- its-registry.unl.edu/unl-its/docker-ci/detect-secrets +- its-registry.unl.edu/unl-its/docker-ci/magento2-unit-test:latest +- its-registry.unl.edu/unl-its/docker-ci/magento2-xml-lint:latest +- its-registry.unl.edu/unl-its/docker-ci/php-lint:5.6 +- its-registry.unl.edu/unl-its/docker-ci/php-lint:7.0 +- its-registry.unl.edu/unl-its/docker-ci/php-lint:7.1 +- its-registry.unl.edu/unl-its/docker-ci/php-lint:7.2 +- its-registry.unl.edu/unl-its/docker-ci/php-unit-test:5.6 +- its-registry.unl.edu/unl-its/docker-ci/php-unit-test:7.0 +- its-registry.unl.edu/unl-its/docker-ci/php-unit-test:7.1 +- its-registry.unl.edu/unl-its/docker-ci/php-unit-test:7.2 +- its-registry.unl.edu/unl-its/docker-ci/static-code-analysis:android +- its-registry.unl.edu/unl-its/docker-ci/static-code-analysis:latest +- its-registry.unl.edu/unl-its/docker-ci/static-code-analysis:php +- its-registry.unl.edu/unl-its/docker-ci/static-code-analysis:python + + + _Last update: Fri May 31 17:38:34 CDT 2019_ diff --git a/_README.md b/_README.md new file mode 100644 index 0000000000000000000000000000000000000000..9f6b66da850c378747d3a896d1d11259f5a5f960 --- /dev/null +++ b/_README.md @@ -0,0 +1,15 @@ +[](https://git.unl.edu/unl-its/docker-ci/commits/develop) + +# Docker-ci + +## Containers usefull in CI process + +### Building Images + +To build these images, clone this repository onto a machine with docker and make installed. Run `make` and all of the images will be built and installed as local docker images. +This file is automatically updated after execute `git commit` based on the content of file `_README.md` To enable this feature, execute `.dev/bootstrap.sh` after cloning the repo. + +### Available tools + +%CONTAINERS_LIST% + diff --git a/dependency-check/Dockerfile b/dependency-check/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..93f3a7f87b25d72b198b600d91dce1842c9a9246 --- /dev/null +++ b/dependency-check/Dockerfile @@ -0,0 +1,36 @@ +FROM adoptopenjdk/openjdk8:alpine + +ARG BUILD_DATE +ARG VCS_REF +ARG VERSION +LABEL org.label-schema.build-date=$BUILD_DATE \ + org.label-schema.name="OWASP Dependency Check" \ + org.label-schema.description="Docker image for dependency checking" \ + org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.vendor="University of Nebraska - Lincoln" \ + org.label-schema.version="0.1.0" \ + org.label-schema.schema-version="1.0" \ + maintainer="J.R. Barreras <rbarrerasmilanes@nebraska.edu>" + +#ENV DEPENDENCY_CHECK_VERSION 4.0.2-release +ENV DEPENDENCY_CHECK_VERSION 5.0.0-M3-release + + +WORKDIR /opt + +RUN apk add --no-cache curl jq su-exec && \ + curl --insecure -o ./dependency-check.zip -L https://dl.bintray.com/jeremy-long/owasp/dependency-check-${DEPENDENCY_CHECK_VERSION}.zip && \ + unzip dependency-check.zip && \ + rm dependency-check.zip && \ + /opt/dependency-check/bin/dependency-check.sh --updateonly + + +ENV PATH $PATH:/opt/dependency-check/bin + +WORKDIR /work + +COPY entrypoint.sh /usr/local/bin/ +RUN chmod +x /usr/local/bin/entrypoint.sh +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] + +CMD ["/usr/local/bin/entrypoint.sh"] diff --git a/dependency-check/entrypoint.sh b/dependency-check/entrypoint.sh new file mode 100644 index 0000000000000000000000000000000000000000..becbcac905f50d5814f51b3b42a10ef072879734 --- /dev/null +++ b/dependency-check/entrypoint.sh @@ -0,0 +1,17 @@ +#!/bin/sh +set -e + +if [ $1 == '-v' ]; then + dependency-check.sh -v + exit 0 +fi + +dependency-check.sh -n --project ${CI_PROJECT_NAMESPACE} -s ./ -f JSON +cat dependency-check-report.json | jq '.dependencies | .[] | select (.vulnerabilities) | [.fileName, .filePath, .vulnerabilities]' | sed -e "s/\"\/work\///g" +RESULT=`cat dependency-check-report.json | jq '.dependencies | .[] | select (.vulnerabilities) | [.fileName, .filePath, .vulnerabilities] | length == 0' | sed -e "s/\"\/work\///g"` + +if [ "${RESULT}" = "" ]; then + exit 0 +else + exit 1 +fi diff --git a/detect-secrets/Dockerfile b/detect-secrets/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..390f8c3581853da6ee43bb8aedfb4b1e2657a559 --- /dev/null +++ b/detect-secrets/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.7-alpine + +ARG BUILD_DATE +ARG VCS_REF +ARG VERSION +LABEL org.label-schema.build-date=$BUILD_DATE \ + org.label-schema.name="Detects secrets" \ + org.label-schema.description="Docker image to detect secrets in a code base" \ + org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.vendor="University of Nebraska - Lincoln" \ + org.label-schema.version="0.1.0" \ + org.label-schema.schema-version="1.0" \ + maintainer="J.R. Barreras <barreras@unl.edu>" + +WORKDIR /work + +RUN apk update && apk upgrade && \ + apk add --no-cache git jq && \ + pip install detect-secrets + +COPY entrypoint.sh /usr/local/bin/ +RUN chmod +x /usr/local/bin/entrypoint.sh +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] + +CMD ["/usr/local/bin/detect-secrets"] diff --git a/detect-secrets/entrypoint.sh b/detect-secrets/entrypoint.sh new file mode 100644 index 0000000000000000000000000000000000000000..0bbe5c0a73f1818f27daf14df44645754d3f4a2f --- /dev/null +++ b/detect-secrets/entrypoint.sh @@ -0,0 +1,34 @@ +#!/bin/sh +set -e + +OPT="" +ENTROPY="4" +while [ -n "$1" ]; do + case "$1" in + -s) + OPT="scan" + ;; + -e) + shift + ENTROPY="$1" + ;; + *) + echo "Invalid parameter(s)" + echo "Usage:" + echo "docker run --rm -v "${PWD}:/work" -w /work its-registry.unl.edu/unl-its/docker-ci/detect-secrets -s -e 3" + echo "" + exit 2 + esac + shift +done + +if [ ${OPT} = "scan" ]; then + OUTPUT=`/usr/local/bin/detect-secrets scan --base64-limit ${ENTROPY}` + echo ${OUTPUT} | jq + RESULT=`echo ${OUTPUT} | jq '.results | length == 0'` + if [ "${RESULT}" = "true" ]; then + exit 0 + else + exit 1 + fi +fi diff --git a/examples/Detect-Secrets.md b/examples/Detect-Secrets.md new file mode 100644 index 0000000000000000000000000000000000000000..49ba574fb07b36f22047029a47a8119f688c737b --- /dev/null +++ b/examples/Detect-Secrets.md @@ -0,0 +1,50 @@ + # Secrets detection example + +## Parameters + +| Argument | Description | +| :------- | :--------------------------- | +| -s | Scans the current directory | +| -e value | Entropy threshold (allowed randomness); Default value 4 | + + +## Detects secrets in the current project + +- One 'analysis' stage with one job +- Allows the job to fail without impacting the rest of the CI (allow_failure: true) + + +``` yml +stages: + - analysis +variables: + stage: analysis + tags: + - docker + script: + - docker run --rm -v "${PWD}:/work" -w /work its-registry.unl.edu/unl-its/docker-ci/detect-secrets -s -e 4.5 + allow_failure: true +``` + +# Inline Allowlisting +To tell detect-secrets to ignore a particular line of code, simply append an inline pragma: allowlist secret comment. For example: + +```python +API_KEY = "blah-blah-but-actually-not-secret" # pragma: allowlist secret +print('hello world') +``` + +Inline commenting syntax for a multitude of languages is supported: + +| Comment Style | Language Support | +| :------------ | :-------------- | +|# | e.g. Python, Dockerfile, YAML | +|// | e.g. Go, C++, Java | +|/* */ | e.g. C, Java | +|' | e.g. Visual Basic .NET | +| -- | e.g. SQL, Haskell | +| <!-- --!> | e.g. XML | + +This may be a convenient way for you to allowlist secrets, without having to regenerate the entire baseline again. Furthermore, this makes the allowlisted secrets easily searchable, auditable, and maintainable. + +_[source](https://github.com/Yelp/detect-secrets#secrets-baseline)_ diff --git a/examples/dependency-check.md b/examples/dependency-check.md new file mode 100644 index 0000000000000000000000000000000000000000..f2160c19218f8f9b759b5c4262c52c017ea96cd3 --- /dev/null +++ b/examples/dependency-check.md @@ -0,0 +1,35 @@ +# Dependecy check example + +## Parameters + +| Argument | Description | +| :------- | :--------------------------- | +| -v | Prints Dependency Check version and exit | +| without parameters | Scans the current directory (must be a git repo) | + + +## Detect outdated dependencies in the current project using [OWASP DependencyCheck](https://www.owasp.org/index.php/OWASP_Dependency_Check) + +- One 'analysis' stage with one job +- Allows the job to fail without impacting the rest of the CI (allow_failure: true) + + +``` yml +stages: + - analysis +variables: + stage: analysis + tags: + - docker + script: + - docker run --rm -v "${PWD}:/work" -w /work its-registry.unl.edu/unl-its/docker-ci/detect-secrets -s -e 4.5 + - docker run --rm -v "${PWD}:/work" -w /work its-registry.unl.edu/unl-its/docker-ci/dependency-check + allow_failure: true +``` + +## Scan the current directory (must be a git repo) + +``` bash +docker run -it --rm -v "${PWD}:/work" -w /work -e CI_PROJECT_NAMESPACE=`basename $(git rev-parse --show-toplevel)` its-registry.unl.edu/unl-its/docker-ci/dependency-check + +```