4lt/docker-compose-cookbooks
1
0
Fork 0
mirror of https://github.com/nickyeoman/docker-compose-cookbooks.git synced 2026-09-04 02:46:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

portainer build script

Browse Source
This commit is contained in:
nick
2025-11-26 20:42:56 -08:00
parent 88f887c5de
commit 20bc9f8c6a
2 changed files with 803 additions and 1116 deletions
Download Patch File Download Diff File Expand all files Collapse all files
_build/portainer_template_build.py
+104 -245
View File
@@ -1,42 +1,13 @@
#!/usr/bin/env python3
"""
Portainer templates generator for nickyeoman/docker-compose-cookbooks
Generates templates.json (Portainer-compatible) based on each project's
docker-compose.yml and sample.env/env-sample.
Behavior:
- Uses master branch for raw GitHub stackfile URLs.
- Skips directories starting with "_" and those ending with "_dev" or "_notes".
- Skips projects whose docker-compose.yml contains "${VOL_PATH".
- Extracts README "## Overview" (exact match) for the description; default fallback.
- Parses sample.env or env-sample, strips inline comments, returns defaults.
- Resolves image lines (handles ${VAR:-default} and ${VAR}) using sample.env defaults.
- Parses ports into strings like "8000:80" or "8080:8080/tcp" for Portainer.
- Parses volumes and converts them into named Docker volumes:
- container path used as container
- named volume created as "<stack>-<container-path-sanitized>" and inserted into "bind"
- Produces a Portainer-friendly JSON structure:
{
"version": "3",
"templates": [
{
"type": "docker-compose",
"title": "<project>",
"name": "<project>",
"note": "...",
"categories": ["Auto"],
"platform": "linux",
"logo": "...",
"description": "...",
"image": "...",
"stackfile": "https://raw.githubusercontent.com/..../master/<project>/docker-compose.yml",
"env": [ {name,label,default}, ... ],
"ports": ["80:80", ...],
"volumes": [ {"container":"/path", "bind":"<named-volume>"}, ... ]
}
]
}
Portainer templates generator (Portainer CE compatible)
Usage:
python3 _build/portainer_template_build.py [--nologo] [--branch BRANCH] [--output PATH] [--no-skip-volpath]
Defaults:
branch = master
output = ./templates.json
skips projects containing "${VOL_PATH" by default (use --no-skip-volpath to disable)
Note: Branch is used only for logo asset lookup; templates use default repo branch (master).
"""
from pathlib import Path
import argparse
@@ -44,15 +15,21 @@ import json
import re
import requests
import sys
# Constants
from typing import List, Dict
# -------------------------
# Config / Defaults
# -------------------------
SCRIPT_DIR = Path(__file__).resolve().parent
REPO_ROOT = SCRIPT_DIR.parent
OUTPUT_FILE = REPO_ROOT / "templates.json"
GITHUB_RAW_BASE = "https://raw.githubusercontent.com/nickyeoman/docker-compose-cookbooks/master"
ASSETS_BASE = "https://i.4lt.ca/cookbook"
SKIP_PAT = "${VOL_PATH"
DEFAULT_OUTPUT = REPO_ROOT / "templates.json"
ASSETS_BASE = "https://i.4lt.ca/cookbooks"
GITHUB_RAW_BASE_TEMPLATE = "https://raw.githubusercontent.com/{owner}/{repo}/{branch}"
GITHUB_OWNER = "nickyeoman"
GITHUB_REPO = "docker-compose-cookbooks"
DEFAULT_BRANCH = "master"
SKIP_VOLPATH_PATTERN = "${VOL_PATH"
# Regex
VAR_RE = re.compile(r"\$\{([^:}]+)(?:[:-]([^}]+))?\}")
# -------------------------
# Helpers
# -------------------------
@@ -61,15 +38,26 @@ def read_text_safe(p: Path) -> str:
return p.read_text()
except Exception:
return ""
def write_atomic(path: Path, data: str):
tmp = path.with_suffix(path.suffix + ".tmp")
tmp.write_text(data)
tmp.replace(path)
def slugify_for_name(s: str) -> str:
# produce a safe component for volume names etc.
s = s.strip().lower()
s = re.sub(r"[^a-z0-9_\-]+", "-", s)
s = re.sub(r"-{2,}", "-", s)
s = s.strip("-")
if not s:
return "x"
return s
# -------------------------
# Extract "Overview" section from README.md (exact "## Overview")
# README Overview extraction
# -------------------------
def extract_overview(readme_path: Path, project_name: str) -> str:
text = read_text_safe(readme_path)
if not text:
return f"{project_name} description to come"
lines = text.splitlines()
overview = []
in_section = False
@@ -86,39 +74,36 @@ def extract_overview(readme_path: Path, project_name: str) -> str:
if not overview:
return f"{project_name} description to come"
return " ".join(overview)
# -------------------------
# Logo URL lookup (skip HEAD if nologo True)
# Logo lookup (HEAD check)
# -------------------------
def find_logo_url(dir_path: Path, nologo: bool = False) -> str:
def find_logo_url(dir_path: Path, nologo: bool, branch: str) -> str:
name = dir_path.name
logo_url = f"{ASSETS_BASE}/{name}.png"
# Use branch for asset if needed, but since assets are static, ignore branch here
logo = f"{ASSETS_BASE}/{name}.png"
default = f"{ASSETS_BASE}/default.png"
if nologo:
return default
try:
resp = requests.head(logo_url, timeout=4)
if resp.status_code == 200:
return logo_url
r = requests.head(logo, timeout=4)
if r.status_code == 200:
return logo
except Exception:
pass
return default
# -------------------------
# Parse env defaults from sample.env or env-sample (strip comments)
# Returns list of dicts { "name": ..., "default": ... }
# Parse environment variables from sample.env/env-sample
# -------------------------
def parse_env_vars(dir_path: Path):
def parse_env_vars(dir_path: Path) -> List[Dict[str,str]]:
candidates = [dir_path / "sample.env", dir_path / "env-sample"]
env_file = None
for p in candidates:
if p.exists():
env_file = p
for c in candidates:
if c.exists():
env_file = c
break
if not env_file:
return []
env_list = []
out = []
for raw in read_text_safe(env_file).splitlines():
line = raw.strip()
if not line or line.startswith("#"):
@@ -126,215 +111,89 @@ def parse_env_vars(dir_path: Path):
if "=" in line:
key, val = line.split("=", 1)
key = key.strip()
# strip inline comments after #
# strip inline comment after #
val = val.split("#", 1)[0].strip()
# remove surrounding quotes
val = val.strip().strip('"').strip("'")
env_list.append({"name": key, "default": val})
return env_list
out.append({"name": key, "default": val})
return out
# -------------------------
# Resolve the first "image:" line from docker-compose.yml
# Supports ${VAR:-default} and ${VAR}
# Build Portainer template object
# -------------------------
VAR_RE = re.compile(r"\$\{([^:}]+)(?:[:-]([^}]+))?\}")
def parse_image(compose_path: Path, env_vars: list) -> str:
text = read_text_safe(compose_path)
if not text:
return ""
env_defaults = {v["name"]: v["default"] for v in env_vars}
for line in text.splitlines():
stripped = line.strip()
if stripped.startswith("image:"):
image = stripped.split("image:", 1)[1].strip().strip('"').strip("'")
# replace ${VAR:-default} or ${VAR} using env_defaults
def repl(m):
var = m.group(1)
default = m.group(2)
return env_defaults.get(var, default or "")
image = VAR_RE.sub(repl, image)
return image
return ""
# -------------------------
# Parse ports into a list of strings Portainer likes
# Examples supported:
# - "8080:80"
# - "9000"
# - "9000:9000/tcp"
# - "53:53/udp"
# -------------------------
def parse_ports(compose_path: Path):
text = read_text_safe(compose_path)
if not text:
return []
ports = []
lines = text.splitlines()
in_ports = False
for line in lines:
stripped = line.strip()
if stripped.startswith("ports:"):
in_ports = True
continue
if in_ports:
if not stripped or not stripped.startswith("-"):
break
# get port string, strip comments/quotes
port_str = stripped[1:].strip().split("#", 1)[0].strip().strip('"').strip("'")
if not port_str:
continue
# normalize space
port_str = port_str.replace(" ", "")
# handle protocol suffix; keep as-is for Portainer
ports.append(port_str)
return ports
# -------------------------
# Parse volumes:
# - Accept mapping forms:
# host:container[:ro|rw]
# container[:ro|rw]
# named-volume:container[:ro|rw]
# - For Portainer templates we'll provide:
# {"container": "/path/in/container", "bind": "<named-volume>"}
# where <named-volume> is generated as "<stack>-<container-path-sanitized>"
# -------------------------
def parse_volumes(compose_path: Path, stack_name: str):
text = read_text_safe(compose_path)
if not text:
return []
vols = []
lines = text.splitlines()
in_volumes = False
for line in lines:
stripped = line.strip()
if stripped.startswith("volumes:"):
in_volumes = True
continue
if in_volumes:
if not stripped or not stripped.startswith("-"):
break
vol_line = stripped[1:].strip().split("#", 1)[0].strip().strip('"').strip("'")
if not vol_line:
continue
# split from right to handle host:container:mode (we want container and ignore host)
parts = vol_line.rsplit(":", 2) # at most 3 parts
# container part is last if there are 2 or more parts, else first
if len(parts) == 1:
container_path = parts[0].strip()
else:
container_path = parts[-2].strip() if len(parts) == 3 else parts[-1].strip()
# explanation:
# - "host:container" -> parts len 2 -> parts[-1] == container
# - "host:container:ro" -> parts len 3 -> parts[-2] == container
# final strip of mode suffix if any left
container_path = re.sub(r":(ro|rw)$", "", container_path)
# ensure container path starts with /
if not container_path.startswith("/"):
# if it's like "config.yaml" or similar, prefix with /
container_path = "/" + container_path.lstrip("/")
# generate a safe named volume
safe = container_path.strip("/").replace("/", "-").replace(".", "-")
if not safe:
safe = "data"
vol_name = f"{stack_name}-{safe}"
vols.append({"container": container_path, "bind": vol_name})
return vols
# -------------------------
# Build a Portainer template object for a project directory
# -------------------------
def generate_template_object(dir_path: Path, nologo: bool = False):
def generate_template_object(dir_path: Path, branch: str, nologo: bool) -> Dict:
name = dir_path.name
compose_file = dir_path / "docker-compose.yml"
readme_file = dir_path / "README.md"
description = extract_overview(readme_file, name)
logo = find_logo_url(dir_path, nologo=nologo)
readme_path = dir_path / "README.md"
env_vars = parse_env_vars(dir_path)
image = parse_image(compose_file, env_vars)
ports = parse_ports(compose_file)
volumes = parse_volumes(compose_file, name)
# env entries: only include those with a name
env_entries = []
for v in env_vars:
if not v.get("name"):
continue
# leave default as empty string if missing (Portainer accepts empty)
env_entries.append({"name": v["name"], "label": v["name"], "default": v.get("default", "")})
# Build stackfile raw URL on master branch
stackfile_url = f"{GITHUB_RAW_BASE}/{name}/docker-compose.yml"
tpl = {
"type": "docker-compose",
obj = {
"type": 3,
"title": name,
"name": name,
"note": "Auto-generated template from repository",
"categories": ["Auto"],
"platform": "linux",
"logo": logo,
"description": description,
# optional convenience field: image
"image": image,
# Portainer expects a full stackfile URL
"stackfile": stackfile_url,
"env": env_entries,
"logo": find_logo_url(dir_path, nologo=nologo, branch=branch),
"description": extract_overview(readme_path, name),
# Git repository for Compose stack
"repository": {
"url": f"https://github.com/{GITHUB_OWNER}/{GITHUB_REPO}",
"stackfile": f"{name}/docker-compose.yml"
}
}
# ports: Portainer accepts list of strings like "8000:80" or "53:53/udp"
if ports:
tpl["ports"] = ports
# volumes: produce list of objects with container and bind (named volume)
if volumes:
tpl["volumes"] = volumes
return tpl
# env
env_entries = []
for v in env_vars:
if not v.get("name"):
continue
env_entries.append({"name": v["name"], "label": v["name"], "default": v.get("default","")})
if env_entries:
obj["env"] = env_entries
return obj
# -------------------------
# Main
# -------------------------
def main():
parser = argparse.ArgumentParser(description="Generate Portainer templates.json from compose cookbooks")
parser.add_argument("--nologo", action="store_true", help="skip HEAD requests for logos (faster, dev)")
parser.add_argument("--nologo", action="store_true", help="Skip HEAD check for logos (faster)")
parser.add_argument("--branch", default=DEFAULT_BRANCH, help="Git branch to use for logo lookup (default: master). Templates use repo default branch.")
parser.add_argument("--output", default=str(DEFAULT_OUTPUT), help="Output path for templates.json")
parser.add_argument("--no-skip-volpath", action="store_true", help="Don't skip projects containing ${VOL_PATH")
args = parser.parse_args()
out_path = Path(args.output).resolve()
branch = args.branch
nologo = args.nologo
skip_volpath = not args.no_skip_volpath
templates = []
for item in REPO_ROOT.iterdir():
skipped = []
errored = []
for item in sorted(REPO_ROOT.iterdir()):
if not item.is_dir():
continue
if item.name.startswith("_"):
continue
if item.name.endswith(("_dev", "_notes")):
continue
compose_path = item / "docker-compose.yml"
if not compose_path.exists():
continue
# skip projects still using VOL_PATH (developer needs to fix)
if SKIP_PAT in read_text_safe(compose_path):
print(f"Skipping {item.name}: still uses ${{VOL_PATH}}")
text = read_text_safe(compose_path)
if skip_volpath and SKIP_VOLPATH_PATTERN in text:
skipped.append((item.name, "uses ${VOL_PATH}"))
continue
tpl = generate_template_object(item, nologo=args.nologo)
templates.append(tpl)
output = {"version": "3", "templates": templates}
# Write atomically if possible
tmp = OUTPUT_FILE.with_suffix(".tmp")
tmp.write_text(json.dumps(output, indent=2))
tmp.replace(OUTPUT_FILE)
print(f"Generated {OUTPUT_FILE} with {len(templates)} templates.")
try:
tpl = generate_template_object(item, branch=branch, nologo=nologo)
templates.append(tpl)
except Exception as e:
errored.append((item.name, str(e)))
output = {"version": "2", "templates": templates}
# write atomic
write_atomic(out_path, json.dumps(output, indent=2))
print(f"Generated {out_path} with {len(templates)} templates.")
if skipped:
print("Skipped projects:")
for n, reason in skipped:
print(f" - {n}: {reason}")
if errored:
print("Errored projects:")
for n, err in errored:
print(f" - {n}: {err}")
if __name__ == "__main__":
main()
main()
templates.json
+699 -871
View File
File diff suppressed because it is too large Load Diff