diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..6313b56c57848efce05faa7aa7e901ccfc2886ea --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..dec1da522875848a1b0de96d20b1d59dd8a7453f --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,42 @@ +stages: + - verify + - build + - test + - security + +Check Makefile: + stage: verify + tags: + - linux, dockerd + script: + - docker run --rm -v "${PWD}:/work" -w /work mandrean/checkmake ./Makefile + +Lint and Validate dockerfiles: + stage: verify + tags: + - linux, dockerd + script: + - make test + +Make All Images: + stage: build + image: projectatomic/dockerfile-lint + tags: + - linux, dockerd + script: + - make + +test: + stage: test + tags: + - dockerd + script: + - docker run --rm unl-its/static-code-analysis sonar-scanner -v + +Scan with Tenable: + stage: security + tags: + - linux, dockerd + script: + - make security + diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..e3ca46a6815af4deac24c5b4e2d154de165e5c0b --- /dev/null +++ b/Makefile @@ -0,0 +1,97 @@ +###################################### +# Build File for Docker Images # +###################################### + +# All Dockerfiles +#################### +DOCKERFILES := $(shell find ./ -name "Dockerfile") + +# List of images to clean +#################### +#IMAGES_TO_CLEAN := $(shell echo $(shell echo $(shell grep -h LABEL $(DOCKERFILES)) | sed -e 's/LABEL name=//g') | tr [:space:] ' \n' | sort -u) +IMAGES_TO_CLEAN := $(addprefix unl-its/, $(shell find ./ -maxdepth 1 -type d -not -name ".*" -printf '%f\n')) + +# Get the latest commit +#################### +GIT_COMMIT = $(strip $(shell git rev-parse --short HEAD)) + +# Docker build arguments +#################### +BUILD_ARGS := --build-arg VCS_REF=$(GIT_COMMIT) + +.PHONY: all clean test $(DOCKERFILES) clean_images $(IMAGES_TO_CLEAN) static-code-analysis delete_dangling_images security + +all: \ + php-lint_5.6 php-lint_7.0 php-lint_7.1 php-lint_7.2 php-lint_latest \ + magento2-unit-test magento2-xml-lint \ + static-code-analysis + +# PHP Images +#################### +php-lint_5.6: php-lint/5.6/Dockerfile + docker build -t unl-its/php-lint:5.6 -f php-lint/5.6/Dockerfile php-lint + +php-lint_7.0: php-lint/7.0/Dockerfile + docker build -t unl-its/php-lint:7.0 -f php-lint/7.0/Dockerfile php-lint + +php-lint_7.1: php-lint/7.1/Dockerfile + docker build -t unl-its/php-lint:7.1 -f php-lint/7.1/Dockerfile php-lint + +php-lint_7.2: php-lint/7.2/Dockerfile + docker build -t unl-its/php-lint:7.2 -f php-lint/7.2/Dockerfile php-lint + +php-lint_latest: php-lint_7.2 + docker tag unl-its/php-lint:7.2 unl-its/php-lint:latest + + +# Application Images +#################### +magento2-unit-test: magento2-unit-test/latest/Dockerfile + docker build -t unl-its/magento2-unit-test:latest magento2-unit-test/latest + +magento2-xml-lint: magento2-unit-test magento2-xml-lint/latest/Dockerfile + docker build -t unl-its/magento2-xml-lint:latest magento2-xml-lint/latest + +# Sonarqube static-code-analysis +# ##################### +static-code-analysis: + docker build $(BUILD_ARGS) -t unl-its/static-code-analysis:latest static-code-analysis/ + docker build $(BUILD_ARGS) -t unl-its/static-code-analysis:php static-code-analysis/ + docker build $(BUILD_ARGS) -t unl-its/static-code-analysis:python static-code-analysis/ + +# Cleanup +#################### +clean: clean_images + +test: $(DOCKERFILES) +$(DOCKERFILES): + @echo "Linting and validating $(@D)..." + @docker run --rm -i nimmis/label-inspector lint < $(@D)/Dockerfile + @docker run --rm -i nimmis/label-inspector validate < $(@D)/Dockerfile + +clean_images: $(IMAGES_TO_CLEAN) delete_dangling_images +$(IMAGES_TO_CLEAN): + @echo "Uninstalling $@" + @docker rmi $(shell docker images --format '{{.Repository}}:{{.Tag}}' --filter reference=$@) + +IMAGES_TO_SCAN = $(shell docker images --format '{{.Repository}}:{{.Tag}}' | grep unl-its ) +security: + @docker login -u ${TENABLE_IO_ACCESS_KEY} -p ${TENABLE_IO_SECRET_KEY} registry.cloud.tenable.com + @for image in $(IMAGES_TO_SCAN); do \ + docker tag $$image registry.cloud.tenable.com/`echo $$image | cut -d "/" -f 2`; \ + docker push registry.cloud.tenable.com/`echo $$image | cut -d "/" -f 2`; \ + done + +delete_dangling_images: + @echo "Removing dangling images" + @docker image prune -f +help: + @echo -e "make [all] \n\t # Creates all the images" + @echo -e "make <TARGET> \n\t # Creates an specific image" + @echo -e "make clean \n\t # Cleanup" + @echo -e "make security \n\t # Send the images to Tenable. Use env variables TENABLE_IO_ACCESS_KEY and TENABLE_IO_SECRET_KEY" + +debug: + @echo -e "DOCKERFILES: \t $(DOCKERFILES)" + @echo -e "IMAGES_TO_CLEAN:\t $(IMAGES_TO_CLEAN)" + @echo -e "IMAGES_TO_SCAN:\t $(IMAGES_TO_SCAN)" diff --git a/README.md b/README.md index 9415342bda6dfc1d81f3aad5989dbe6aa0bc19ed..995a52e832bc53cdca5c6e566c1ef646defbcea2 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ # docker-ci +Containers used on CI -Containers used on CI \ No newline at end of file +## Building Images +To build these images, clone this repository onto a machine with docker and make installed. Run `make` and all of the images will be built and installed as local docker images. diff --git a/magento2-unit-test/latest/Dockerfile b/magento2-unit-test/latest/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..23b6c24c3a6877afe78476bd287fdce23e275659 --- /dev/null +++ b/magento2-unit-test/latest/Dockerfile @@ -0,0 +1,49 @@ +FROM centos:7 + +ARG BUILD_DATE +ARG VCS_REF +ARG VERSION +LABEL org.label-schema.build-date=$BUILD_DATE \ + org.label-schema.name="Magento Unit Tester" \ + org.label-schema.description="" \ + org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.vendor="University of Nebraska - Lincoln" \ + org.label-schema.version=$VERSION \ + org.label-schema.schema-version="1.0" \ + maintainer="Alan Nelson <alan.nelson@nebraska.edu>" + + +# Add additional REPOs +RUN rpm -i https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm \ + && rpm -i https://rhel7.iuscommunity.org/ius-release.rpm + +# Update system and install required packages +RUN yum update -y \ + && yum install -y \ + unzip \ + php71u-cli \ + php71u-json \ + php71u-pdo \ + php71u-mysqlnd \ + php71u-opcache \ + php71u-xml \ + php71u-mcrypt \ + php71u-gd \ + php71u-devel \ + php71u-intl \ + php71u-mbstring \ + php71u-bcmath \ + php71u-json \ + php71u-soap \ + php71u-pecl-xdebug \ + && yum clean all \ + && rm -rf /var/cache/yum + +# Install composer +RUN curl -o /usr/local/bin/composer https://getcomposer.org/composer.phar \ + && chmod 755 /usr/local/bin/composer + +# PHP Config file +COPY php.ini /etc/php.d/mg2.ini + +CMD ["bash"] diff --git a/magento2-unit-test/latest/php.ini b/magento2-unit-test/latest/php.ini new file mode 100644 index 0000000000000000000000000000000000000000..c8a7ade396fe6860154ef560364bc74abf4384c3 --- /dev/null +++ b/magento2-unit-test/latest/php.ini @@ -0,0 +1,5 @@ +memory_limit = 2G + +session.auto_start = off; + +date.timezone="America/Chicago" diff --git a/magento2-xml-lint/latest/Dockerfile b/magento2-xml-lint/latest/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..b7d5072b9930e4ee66e6fbfed5c15825dd53bfea --- /dev/null +++ b/magento2-xml-lint/latest/Dockerfile @@ -0,0 +1,21 @@ +FROM unl-its/magento2-unit-test:latest + +ARG BUILD_DATE +ARG VCS_REF +ARG VERSION +LABEL org.label-schema.build-date=$BUILD_DATE \ + org.label-schema.name="Magento 2 XML linter" \ + org.label-schema.description="" \ + org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.vendor="University of Nebraska - Lincoln" \ + org.label-schema.version=$VERSION \ + org.label-schema.schema-version="1.0" \ + maintainer="Alan Nelson <alan.nelson@nebraska.edu>" + +# Scripts +COPY xml-lint docker-entrypoint /usr/local/bin/ + +# Permissions +RUN chmod 755 /usr/local/bin/xml-lint /usr/local/bin/docker-entrypoint + +ENTRYPOINT ["docker-entrypoint"] diff --git a/magento2-xml-lint/latest/docker-entrypoint b/magento2-xml-lint/latest/docker-entrypoint new file mode 100644 index 0000000000000000000000000000000000000000..0cd1ad3743a8510f070eb8c303d1ea5c0ac8e9a0 --- /dev/null +++ b/magento2-xml-lint/latest/docker-entrypoint @@ -0,0 +1,8 @@ +#!/bin/bash +set -e + +if [ "${1#-}" != "$1" ]; then + set -- bash "$@" +fi + +exec "$@" diff --git a/magento2-xml-lint/latest/xml-lint b/magento2-xml-lint/latest/xml-lint new file mode 100644 index 0000000000000000000000000000000000000000..5e5b32a6de05d33aa510e33fd518891faed2df24 --- /dev/null +++ b/magento2-xml-lint/latest/xml-lint @@ -0,0 +1,81 @@ +#!/usr/bin/python + +import os.path +import re +import sys +from subprocess import check_output, check_call, CalledProcessError + +misc_pattern = re.compile('<resource\s*url="(.+?)"\s*location="(.+?)"\s+\/>') +xsd_pattern = re.compile('xsi:noNamespaceSchemaLocation="(.+?)"') + +def get_mappings(misc_file, base_dir): + mapping = {} + for line in open(misc_file): + result = misc_pattern.search(line) + if result is not None: + mapping[result.group(1)] = result.group(2).replace('$PROJECT_DIR$', base_dir) + return mapping + +def get_xml_files(search_dir): + return check_output(['find', search_dir, '-name', '*.xml']).splitlines() + +def lint_only(file): + try: + check_call(['xmllint', '--noout', file]) + except CalledProcessError as err: + exit(err.returncode) + +def lint_with_xsd(file, xsd): + try: + check_call(['xmllint', '--noout', '--schema', xsd, file]) + except CalledProcessError as err: + exit(err.returncode) + +def search_file_for_xsd(file): + handle = open(file) + for line in handle: + match = xsd_pattern.search(line) + if match is not None: + handle.close() + return match.group(1) + handle.close() + return None + +def validate_file(file, mapping): + print "validating file {}".format(file) + xsd = search_file_for_xsd(file) + if xsd is not None: + if xsd in mapping: + lint_with_xsd(file, mapping[xsd]) + else: + print 'WARNING: Unable to map XSD to path: {}'.format(xsd) + else: + print 'WARNING: Unable to find XSD for file: {}'.format(file) + lint_only(file) + +def main(): + if len(sys.argv) < 4: + print "Usage: xml-lint <misc.xml> <app_base_dir> <search_dir>" + exit(1) + + misc_file = sys.argv[1] + base_dir = os.path.abspath(sys.argv[2]) + search_dir = os.path.abspath(sys.argv[3]) + + if not os.path.isfile(misc_file): + print "{} is not a file".format(misc_file) + exit(1) + + mapping = get_mappings(misc_file, base_dir) + print "loaded {} XSD mapping(s)".format(len(mapping)) + + xml_files = get_xml_files(search_dir) + print "found {} XML file(s)".format(len(xml_files)) + + print "" + for file in xml_files: + validate_file(file, mapping) + print "" + +if __name__ == '__main__': + main() diff --git a/php-lint/5.6/Dockerfile b/php-lint/5.6/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..26c505d6a933b9e0b813fdf7cd8499670c6e6a70 --- /dev/null +++ b/php-lint/5.6/Dockerfile @@ -0,0 +1,21 @@ +FROM php:5.6-alpine + +ARG BUILD_DATE +ARG VCS_REF +ARG VERSION +LABEL org.label-schema.build-date=$BUILD_DATE \ + org.label-schema.name="PHP 5.6 linter" \ + org.label-schema.description="" \ + org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.vendor="University of Nebraska - Lincoln" \ + org.label-schema.version=$VERSION \ + org.label-schema.schema-version="1.0" \ + maintainer="Alan Nelson <alan.nelson@nebraska.edu>" + +RUN apk add --no-cache bash + +COPY php-lint docker-entrypoint /usr/local/bin/ +RUN chmod 755 /usr/local/bin/docker-entrypoint /usr/local/bin/php-lint + +ENTRYPOINT ["docker-entrypoint"] +CMD ["--help"] diff --git a/php-lint/7.0/Dockerfile b/php-lint/7.0/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..4803f6136ab4edada23fe374a6e6b3372da34201 --- /dev/null +++ b/php-lint/7.0/Dockerfile @@ -0,0 +1,21 @@ +FROM php:7.0-alpine + +ARG BUILD_DATE +ARG VCS_REF +ARG VERSION +LABEL org.label-schema.build-date=$BUILD_DATE \ + org.label-schema.name="PHP 7.0 linter" \ + org.label-schema.description="" \ + org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.vendor="University of Nebraska - Lincoln" \ + org.label-schema.version=$VERSION \ + org.label-schema.schema-version="1.0" \ + maintainer="Alan Nelson <alan.nelson@nebraska.edu>" + +RUN apk add --no-cache bash + +COPY php-lint docker-entrypoint /usr/local/bin/ +RUN chmod 755 /usr/local/bin/docker-entrypoint /usr/local/bin/php-lint + +ENTRYPOINT ["docker-entrypoint"] +CMD ["--help"] diff --git a/php-lint/7.1/Dockerfile b/php-lint/7.1/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..7bdd3b4f2ab5ad8c603249a475645feae6a6cf04 --- /dev/null +++ b/php-lint/7.1/Dockerfile @@ -0,0 +1,21 @@ +FROM php:7.1-alpine + +ARG BUILD_DATE +ARG VCS_REF +ARG VERSION +LABEL org.label-schema.build-date=$BUILD_DATE \ + org.label-schema.name="PHP 7.1 linter" \ + org.label-schema.description="" \ + org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.vendor="University of Nebraska - Lincoln" \ + org.label-schema.version=$VERSION \ + org.label-schema.schema-version="1.0" \ + maintainer="Alan Nelson <alan.nelson@nebraska.edu>" + +RUN apk add --no-cache bash + +COPY php-lint docker-entrypoint /usr/local/bin/ +RUN chmod 755 /usr/local/bin/docker-entrypoint /usr/local/bin/php-lint + +ENTRYPOINT ["docker-entrypoint"] +CMD ["--help"] diff --git a/php-lint/7.2/Dockerfile b/php-lint/7.2/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..fcc2d84eda1f9256591694657410f3d40339a10e --- /dev/null +++ b/php-lint/7.2/Dockerfile @@ -0,0 +1,21 @@ +FROM php:7.2-alpine + +ARG BUILD_DATE +ARG VCS_REF +ARG VERSION +LABEL org.label-schema.build-date=$BUILD_DATE \ + org.label-schema.name="PHP 7.2 linter" \ + org.label-schema.description="" \ + org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.vendor="University of Nebraska - Lincoln" \ + org.label-schema.version=$VERSION \ + org.label-schema.schema-version="1.0" \ + maintainer="Alan Nelson <alan.nelson@nebraska.edu>" + +RUN apk add --no-cache bash + +COPY php-lint docker-entrypoint /usr/local/bin/ +RUN chmod 755 /usr/local/bin/docker-entrypoint /usr/local/bin/php-lint + +ENTRYPOINT ["docker-entrypoint"] +CMD ["--help"] diff --git a/php-lint/docker-entrypoint b/php-lint/docker-entrypoint new file mode 100644 index 0000000000000000000000000000000000000000..131b834a0cec520bad3f490a0b746409685ad973 --- /dev/null +++ b/php-lint/docker-entrypoint @@ -0,0 +1,8 @@ +#!/bin/bash +set -e + +if [ "${1#-}" != "$1" ]; then + set -- php-lint "$@" +fi + +exec "$@" diff --git a/php-lint/php-lint b/php-lint/php-lint new file mode 100644 index 0000000000000000000000000000000000000000..afb52f2337d64b0c7e1b97b4daf665909e744329 --- /dev/null +++ b/php-lint/php-lint @@ -0,0 +1,66 @@ +#!/bin/bash + +# Read CLI options +LINT_DIRS=() +LINT_EXTS=() +QUIET=false +while [[ $# -gt 0 ]]; do + case $1 in + -e|--ext) + LINT_EXTS+=("$2") + shift + shift + ;; + -d|--dir) + LINT_DIRS+=("$2") + shift + shift + ;; + -q|--quiet) + QUIET=true + shift + ;; + -h|--help) + printf 'Usage: %s <options>\n' "$0" + printf '\t%s\n' "-e,--ext: Required argument: extensions to lint" + printf '\t%s\n' "-d,--dir: Required argument: directories to lint" + printf '\t%s\n' "-q,--quiet: Optional argument: Quiet mode, only print errors" + printf '\t%s\n' "-h,--help: Prints this help message" + exit 0 + ;; + *) + echo "Unrecognized option ${1}" + shift + ;; + esac +done + +# If no dirs were provided, use CWD +if [[ -z "$LINT_DIRS" ]]; then + LINT_DIRS=($(pwd)) +fi + +# If no extenions were provided, use reasonable defaults +if [[ -z "$LINT_EXTS" ]]; then + LINT_EXTS=(".php" ".phtml") +fi + +# Run PHP Lint on all provided files and directories +for dir in "${LINT_DIRS[@]}"; do + for ext in "${LINT_EXTS[@]}"; do + echo "Scanning directory ${dir} for extension ${ext}" + + # Scan current dir and ext and lint them + for f in $(find "${dir}" -type f -name "*${ext}"); do + OUTPUT=$(php -l $f 2>&1) + rc=$? + + if [[ $rc != 0 ]]; then # Non-zero exit code, print error and exit + >&2 echo "$OUTPUT" + exit $rc + elif [[ $rc == 0 ]] && [[ $QUIET == false ]]; then # all ok + echo "$OUTPUT" + fi + done + done +done diff --git a/static-code-analysis/Dockerfile b/static-code-analysis/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..afa9f95a5f370769c596cf1ad21b80dfabfbb4ac --- /dev/null +++ b/static-code-analysis/Dockerfile @@ -0,0 +1,33 @@ +FROM openjdk:8-jre-alpine + +ARG BUILD_DATE +ARG VCS_REF +ARG VERSION +LABEL org.label-schema.build-date=$BUILD_DATE \ + org.label-schema.name="Sonarqube Static Code Analyzer" \ + org.label-schema.description="Docker image for static code analysis using a Sonarqube server" \ + org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.vendor="University of Nebraska - Lincoln" \ + org.label-schema.version="0.1.0" \ + org.label-schema.schema-version="1.0" \ + maintainer="J.R. Barreras <barreras@unl.edu>" + +ENV SONAR_SCANNER_VERSION 3.2.0.1227 + +WORKDIR /opt + +RUN apk add --no-cache curl sed bash nodejs-current nodejs-npm su-exec && \ + mkdir -p /opt/src && \ + curl --insecure -o ./sonarscanner.zip -L https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-${SONAR_SCANNER_VERSION}.zip && \ + unzip sonarscanner.zip && \ + rm sonarscanner.zip + +ENV SONAR_RUNNER_HOME=/opt/sonar-scanner-${SONAR_SCANNER_VERSION} +ENV PATH $PATH:/opt/sonar-scanner-${SONAR_SCANNER_VERSION}/bin + +COPY entrypoint.sh /usr/local/bin/ +RUN chmod +x /usr/local/bin/entrypoint.sh +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] + +CMD /bin/bash + diff --git a/static-code-analysis/entrypoint.sh b/static-code-analysis/entrypoint.sh new file mode 100644 index 0000000000000000000000000000000000000000..ae5005ecc0cce43e92b0deef7f09b317e36d55e1 --- /dev/null +++ b/static-code-analysis/entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# Add local user +# Either use the LOCAL_USER_ID if passed in at runtime or +# fallback + +USER_ID=${LOCAL_USER_ID:-9001} + +su-exec user "$@"