diff --git a/.gitignore b/.gitignore index 6cc2991..26d9aab 100644 --- a/.gitignore +++ b/.gitignore @@ -28,5 +28,8 @@ ansible.log # Local and CI-generated artifacts artifacts/ +# CI-generated temporary secret material +.run/ + # Local reference material not consumed by the pipeline source_documents/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9722d4d..b9c8da6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -109,10 +109,14 @@ test:ansible: name: "$TARGET_ENVIRONMENT" action: verify script: - - test -n "${ANSIBLE_SECRET_VARS:-}" || { echo "ANSIBLE_SECRET_VARS file variable is required" >&2; exit 1; } - - test -f "$ANSIBLE_SECRET_VARS" || { echo "ANSIBLE_SECRET_VARS must be a GitLab file variable" >&2; exit 1; } - - export ANSIBLE_VARS_FILE="$ANSIBLE_SECRET_VARS" - - bash methodologies/ansible/run.sh --limit "${ANSIBLE_LIMIT:-all}" + - | + SECRETS_DIR="$CI_PROJECT_DIR/.run/secrets" + python3 methodologies/ansible/scripts/build-secrets.py --out-dir "$SECRETS_DIR" + export ANSIBLE_VARS_FILE="$SECRETS_DIR/ansible-vars.yml" + if [ -f "$SECRETS_DIR/ansible-private-key" ]; then + export ANSIBLE_PRIVATE_KEY_FILE="$SECRETS_DIR/ansible-private-key" + fi + bash methodologies/ansible/run.sh --limit "${ANSIBLE_LIMIT:-all}" artifacts: when: always expire_in: 90 days diff --git a/README.md b/README.md index 32202bf..dfc9a54 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,13 @@ The pipeline validates that `TARGET_ENVIRONMENT` matches `assets.yml`. A mismatc ## Secrets -The Ansible job requires `ANSIBLE_SECRET_VARS` as an environment-scoped GitLab **File** variable containing Ansible variables. SSH keys may be supplied as `ANSIBLE_PRIVATE_KEY_FILE`, also as a File variable. +The Ansible job consumes credentials as individual environment-scoped GitLab +**Variable** entries — `ANSIBLE_USER`, `ANSIBLE_PASSWORD`, +`ANSIBLE_BECOME_PASSWORD`, `VCENTER_HOSTNAME`, `VCENTER_USERNAME`, and +`VCENTER_PASSWORD` — plus an optional `ANSIBLE_PRIVATE_KEY` **File** variable +for SSH key authentication. See [instructions.md](instructions.md) for a +step-by-step setup guide and [docs/secrets.md](docs/secrets.md) for variable +examples, masking guidance, and rotation. Validation, normalization, and report jobs do not declare a GitLab environment and therefore do not receive environment-scoped credentials. See [docs/secrets.md](docs/secrets.md) for variable examples and rotation guidance. diff --git a/docs/secrets.md b/docs/secrets.md index 2e89928..a92c60f 100644 --- a/docs/secrets.md +++ b/docs/secrets.md @@ -32,29 +32,37 @@ Do not enable `CI_DEBUG_TRACE` in pipelines that receive secrets. ## Ansible Variables -Create `ANSIBLE_SECRET_VARS` as an environment-scoped **File** variable. Its -contents are an Ansible YAML or JSON variables file, for example: +Define credentials as individual environment-scoped **Variable** entries (not a +single File variable), under **Settings > CI/CD > Variables**: -```yaml -ansible_user: "DOMAIN\\automation-user" -ansible_password: "replace-in-gitlab" -ansible_become_password: "replace-in-gitlab" -vcenter_username: "automation@vsphere.local" -vcenter_password: "replace-in-gitlab" -``` +| GitLab variable | Ansible variable | Purpose | +| --- | --- | --- | +| `ANSIBLE_USER` | `ansible_user` | Login account | +| `ANSIBLE_PASSWORD` | `ansible_password` | Login password | +| `ANSIBLE_BECOME_PASSWORD` | `ansible_become_password` | Sudo / enable password | +| `VCENTER_HOSTNAME` | `vcenter_hostname` | vCenter appliance address | +| `VCENTER_USERNAME` | `vcenter_username` | vCenter account | +| `VCENTER_PASSWORD` | `vcenter_password` | vCenter password | -The `test:ansible` job checks that the variable resolves to a file and exports -its temporary path as `ANSIBLE_VARS_FILE`. The methodology runner passes it with -`--extra-vars @` without printing the contents or putting values in the -process command line. +Set only those required by the target types declared in `assets.yml`. Values +containing spaces or special characters (for example a Windows `DOMAIN\user` +password) must use **Masked and hidden** (GitLab 15.10+); standard **Masked** +rejects such formats. -For SSH key authentication, add `ANSIBLE_PRIVATE_KEY_FILE` as a separate -environment-scoped **File** variable. The methodology runner passes that path through -`--private-key` when present. +The `test:ansible` job runs `methodologies/ansible/scripts/build-secrets.py`, +which maps these variables to their Ansible names and writes a temporary YAML +vars file. `run.sh` passes that file with `--extra-vars @` without +printing the contents or putting values in the process command line. The +temporary files live under `$CI_PROJECT_DIR/.run/secrets/` and are removed when +the job pod ends; they are never stored in artifacts. + +For SSH key authentication, add `ANSIBLE_PRIVATE_KEY` as a separate +environment-scoped **File** variable. The job writes its contents to a `0600` +temporary key file and passes that path through `--private-key` when present. The current job assumes one credential set per test environment. Environments with distinct Windows, Linux, network, or hypervisor credentials should be split -into separate tool jobs, each referencing its own scoped File variable. +into separate tool jobs, each referencing its own environment-scoped variables. ## ZAP Variables diff --git a/instructions.md b/instructions.md new file mode 100644 index 0000000..e00dec5 --- /dev/null +++ b/instructions.md @@ -0,0 +1,137 @@ +# Test Execution Instructions + +This guide walks a tester through configuring a test environment, defining its +secrets, and running the compliance pipeline in GitLab CI. Secrets are supplied +as individual GitLab CI/CD variables; no secrets file is committed to the +repository. + +--- + +## 1. Define the project and targets in `assets.yml` + +`assets.yml` is the single source of non-secret project and target data. It is +both an Ansible inventory and the input GitLab CI validates. + +### Project metadata + +Under `all.vars.test_project`, set: + +```yaml +all: + vars: + test_project: + id: "baggage-handling" # unique project identifier + name: "Baggage Handling HLC" # display name for reports + environment: "test" # must match TARGET_ENVIRONMENT + customer: "Customer X" + location: "testserv" +``` + +> The `environment` value is the GitLab environment scope that selects the +> matching secrets. `test`, `acceptance`, and `production` are typical values. + +### Targets + +Add each host under the matching group in `all.children`, for example: + +```yaml + linux_vms: + hosts: + linux-app-01: + ansible_host: 192.0.2.10 + asset_type: linux_vm + test_profiles: [iec62443, sbom] +``` + +Supported groups and their required connection settings are already defined in +`assets.yml`: + +| Group | Connection | Example secret variables needed | +| --- | --- | --- | +| `linux_vms` | SSH | `ANSIBLE_USER`, `ANSIBLE_PASSWORD`, `ANSIBLE_BECOME_PASSWORD` | +| `windows_servers` / `windows_clients` / `mssql_servers` / `hyperv_hosts` | WinRM | `ANSIBLE_USER`, `ANSIBLE_PASSWORD` | +| `vmware_esxi` | vCenter API | `VCENTER_HOSTNAME`, `VCENTER_USERNAME`, `VCENTER_PASSWORD` | +| `cisco_switches` / `cisco_firewalls` | network_cli | `ANSIBLE_USER`, `ANSIBLE_PASSWORD`, `ANSIBLE_BECOME_PASSWORD` | + +Keep credentials out of this file. + +--- + +## 2. Define secrets as GitLab CI/CD variables + +Open the project in GitLab: **Settings → CI/CD → Variables → Add variable**. + +Add one variable per secret. Set the **Environment scope** to the exact value of +`all.vars.test_project.environment` (for example `test`), so secrets are only +injected into tool jobs that declare that environment. + +| Key | Type | Protected | Masking | Notes | +| --- | --- | --- | --- | --- | +| `ANSIBLE_USER` | Variable | recommended | Masked | Login account, e.g. `DOMAIN\automation-user` | +| `ANSIBLE_PASSWORD` | Variable | yes | Masked and hidden | Login password | +| `ANSIBLE_BECOME_PASSWORD` | Variable | yes | Masked and hidden | Sudo/enable password (Linux, Cisco) | +| `VCENTER_HOSTNAME` | Variable | recommended | Masked | vCenter appliance, e.g. `vcenter.example.com` | +| `VCENTER_USERNAME` | Variable | recommended | Masked | e.g. `automation@vsphere.local` | +| `VCENTER_PASSWORD` | Variable | yes | Masked and hidden | vCenter password | +| `ANSIBLE_PRIVATE_KEY` | File | yes | — | Optional: SSH private key for key auth | + +Rules: + +- Define **only the variables required by the targets in `assets.yml`**. +- Use **Masked and hidden** (GitLab 15.10+) for values containing spaces or + special characters; standard **Masked** rejects those formats. +- Do not enable `CI_DEBUG_TRACE` on pipelines that receive secrets. + +The `test:ansible` job runs `methodologies/ansible/scripts/build-secrets.py`, +which assembles these variables into a temporary vars file and passes it to +Ansible via `--extra-vars @`. Values never appear in the job log or on the +process command line. + +--- + +## 3. Run the pipeline + +1. Push your `assets.yml` changes (and any repository changes) to the branch you + want to test. +2. In GitLab, open **CI/CD → Pipelines → Run pipeline** (or **New pipeline**). +3. Set the pipeline variables: + +| Variable | Value | Purpose | +| --- | --- | --- | +| `TARGET_ENVIRONMENT` | `test` (or your environment) | Selects the environment scope; must match `assets.yml` | +| `RUN_ANSIBLE` | `true` | Enables the infrastructure tests | +| `RUN_ZAP` | `true` | Enables web tests (only once the ZAP methodology is enabled) | +| `ANSIBLE_LIMIT` | e.g. `linux_vms` or `linux-app-01` | Optional: restrict the run to a host or group (defaults to `all`) | + +4. Start the pipeline. + +The pipeline validates that `TARGET_ENVIRONMENT` matches `assets.yml` before any +testing begins. A mismatch stops the run in the `validate` stage. + +--- + +## 4. Check the results + +Pipeline stages: `validate → build → platform → test → normalize → report`. + +- `test:ansible` runs the compliance checks against the selected assets. +- `normalize` converts the tool output into the common schema. +- `report` produces Markdown, HTML, and PDF. + +Download artifacts from the pipeline page (**Download artifacts** on the job, or +the pipeline's **Artifacts** menu): + +```text +artifacts/ +├── raw/ansible/ native Ansible output +├── normalized/ common JSON schema +└── rendered/ Markdown, HTML, and PDF reports +``` + +--- + +## 5. Rotating a secret + +Replace the value of the GitLab CI/CD variable under the matching environment +scope. No repository change is required. Existing artifacts contain only +normalized findings, never the secret values. diff --git a/methodologies/ansible/README.md b/methodologies/ansible/README.md index 0b4ece1..9ccf8d5 100644 --- a/methodologies/ansible/README.md +++ b/methodologies/ansible/README.md @@ -18,6 +18,10 @@ GitLab runs this methodology when `RUN_ANSIBLE=true`. For local use from the rep ANSIBLE_VARS_FILE=/secure/vars.yml ./methodologies/ansible/run.sh --limit linux_vms ``` +In CI, `scripts/build-secrets.py` assembles individual GitLab variables +(`ANSIBLE_USER`, `ANSIBLE_PASSWORD`, ...) into that vars file, so a committed +secrets file is never required. + Build the image with: ```bash diff --git a/methodologies/ansible/scripts/build-secrets.py b/methodologies/ansible/scripts/build-secrets.py new file mode 100644 index 0000000..aff9864 --- /dev/null +++ b/methodologies/ansible/scripts/build-secrets.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Assemble individual GitLab CI/CD variables into a temporary Ansible vars file. + +Reads a fixed allowlist of environment variables (injected by GitLab from +masked CI/CD variables), maps them to their Ansible names, and writes them to a +YAML vars file for ``--extra-vars @``. An optional private key is written +to a separate 0600 file for ``--private-key``. + +Values never appear on the process command line or in this script's output. +Only the allowlisted variable names are consumed; everything else is ignored. +""" + +import argparse +import os +import stat +from pathlib import Path + +import yaml + +# GitLab variable name -> Ansible variable name. Extend this mapping when a new +# connection variable needs to be supplied per environment. +VAR_MAP = { + "ANSIBLE_USER": "ansible_user", + "ANSIBLE_PASSWORD": "ansible_password", + "ANSIBLE_BECOME_PASSWORD": "ansible_become_password", + "VCENTER_HOSTNAME": "vcenter_hostname", + "VCENTER_USERNAME": "vcenter_username", + "VCENTER_PASSWORD": "vcenter_password", +} + +PRIVATE_KEY_ENV = "ANSIBLE_PRIVATE_KEY" +VARS_FILENAME = "ansible-vars.yml" +KEY_FILENAME = "ansible-private-key" + + +def write_secret_file(path: Path, data: bytes) -> None: + path.write_bytes(data) + path.chmod(stat.S_IRUSR | stat.S_IWUSR) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--out-dir", + type=Path, + required=True, + help="Directory for the generated vars file and private key", + ) + args = parser.parse_args() + + args.out_dir.mkdir(parents=True, exist_ok=True) + os.chmod(args.out_dir, stat.S_IRWXU) + + variables = { + ansible_name: os.environ[gitlab_name] + for gitlab_name, ansible_name in VAR_MAP.items() + if os.environ.get(gitlab_name) + } + + private_key = os.environ.get(PRIVATE_KEY_ENV) + if not variables and not private_key: + raise SystemExit( + "no Ansible secrets were provided. Define at least one of " + + ", ".join(VAR_MAP) + + f" or {PRIVATE_KEY_ENV} as an environment-scoped CI/CD variable." + ) + + vars_file = args.out_dir / VARS_FILENAME + write_secret_file( + vars_file, + yaml.safe_dump(variables, sort_keys=True, default_flow_style=False).encode( + "utf-8" + ), + ) + print(f"Wrote Ansible variables to {vars_file}") + + if private_key: + key_data = ( + Path(private_key).read_bytes() + if os.path.isfile(private_key) + else private_key.encode("utf-8") + ) + key_file = args.out_dir / KEY_FILENAME + write_secret_file(key_file, key_data) + print(f"Wrote private key to {key_file}") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())