new portainer build script

_build/portainer_template_build.py

@@ -1,28 +1,78 @@
 #!/usr/bin/env python3
-import os
+"""
+Portainer templates generator for nickyeoman/docker-compose-cookbooks
+
+Generates templates.json (Portainer-compatible) based on each project's
+docker-compose.yml and sample.env/env-sample.
+
+Behavior:
+ - Uses master branch for raw GitHub stackfile URLs.
+ - Skips directories starting with "_" and those ending with "_dev" or "_notes".
+ - Skips projects whose docker-compose.yml contains "${VOL_PATH".
+ - Extracts README "## Overview" (exact match) for the description; default fallback.
+ - Parses sample.env or env-sample, strips inline comments, returns defaults.
+ - Resolves image lines (handles ${VAR:-default} and ${VAR}) using sample.env defaults.
+ - Parses ports into strings like "8000:80" or "8080:8080/tcp" for Portainer.
+ - Parses volumes and converts them into named Docker volumes:
+     - container path used as container
+     - named volume created as "<stack>-<container-path-sanitized>" and inserted into "bind"
+ - Produces a Portainer-friendly JSON structure:
+     {
+       "version": "3",
+       "templates": [
+         {
+           "type": "docker-compose",
+           "title": "<project>",
+           "name": "<project>",
+           "note": "...",
+           "categories": ["Auto"],
+           "platform": "linux",
+           "logo": "...",
+           "description": "...",
+           "image": "...",
+           "stackfile": "https://raw.githubusercontent.com/..../master/<project>/docker-compose.yml",
+           "env": [ {name,label,default}, ... ],
+           "ports": ["80:80", ...],
+           "volumes": [ {"container":"/path", "bind":"<named-volume>"}, ... ]
+         }
+       ]
+     }
+"""
+from pathlib import Path
+import argparse
 import json
 import re
 import requests
-from pathlib import Path
+import sys
 
-# ---------------------------------------------------------
+# Constants
-# Paths
-# ---------------------------------------------------------
 SCRIPT_DIR = Path(__file__).resolve().parent
 REPO_ROOT = SCRIPT_DIR.parent
 OUTPUT_FILE = REPO_ROOT / "templates.json"
+GITHUB_RAW_BASE = "https://raw.githubusercontent.com/nickyeoman/docker-compose-cookbooks/master"
+ASSETS_BASE = "https://i.4lt.ca/cookbook"
+SKIP_PAT = "${VOL_PATH"
 
-# ---------------------------------------------------------
+# -------------------------
-# Extract "Overview" section from README.md (strict)
+# Helpers
-# ---------------------------------------------------------
+# -------------------------
+def read_text_safe(p: Path) -> str:
+    try:
+        return p.read_text()
+    except Exception:
+        return ""
+
+# -------------------------
+# Extract "Overview" section from README.md (exact "## Overview")
+# -------------------------
 def extract_overview(readme_path: Path, project_name: str) -> str:
-    if not readme_path.exists():
+    text = read_text_safe(readme_path)
+    if not text:
         return f"{project_name} description to come"
 
-    lines = readme_path.read_text().splitlines()
+    lines = text.splitlines()
     overview = []
     in_section = False
-
     for line in lines:
         stripped = line.strip()
         if stripped == "## Overview":
@@ -33,69 +83,98 @@ def extract_overview(readme_path: Path, project_name: str) -> str:
                 break
             if stripped:
                 overview.append(stripped)
-
     if not overview:
         return f"{project_name} description to come"
-
     return " ".join(overview)
 
-# ---------------------------------------------------------
+# -------------------------
-# Logo URL mapping with existence check
+# Logo URL lookup (skip HEAD if nologo True)
-# ---------------------------------------------------------
+# -------------------------
 def find_logo_url(dir_path: Path, nologo: bool = False) -> str:
-    base_url = "https://i.4lt.ca/cookbooks/"
     name = dir_path.name
-    logo_url = f"{base_url}{name}.png"
+    logo_url = f"{ASSETS_BASE}/{name}.png"
-
+    default = f"{ASSETS_BASE}/default.png"
     if nologo:
-        return f"{base_url}default.png"
+        return default
-
     try:
-        resp = requests.head(logo_url, timeout=5)
+        resp = requests.head(logo_url, timeout=4)
         if resp.status_code == 200:
             return logo_url
-    except requests.RequestException:
+    except Exception:
         pass
+    return default
 
-    return f"{base_url}default.png"
+# -------------------------
+# Parse env defaults from sample.env or env-sample (strip comments)
+# Returns list of dicts { "name": ..., "default": ... }
+# -------------------------
+def parse_env_vars(dir_path: Path):
+    candidates = [dir_path / "sample.env", dir_path / "env-sample"]
+    env_file = None
+    for p in candidates:
+        if p.exists():
+            env_file = p
+            break
+    if not env_file:
+        return []
+
+    env_list = []
+    for raw in read_text_safe(env_file).splitlines():
+        line = raw.strip()
+        if not line or line.startswith("#"):
+            continue
+        if "=" in line:
+            key, val = line.split("=", 1)
+            key = key.strip()
+            # strip inline comments after #
+            val = val.split("#", 1)[0].strip()
+            # remove surrounding quotes
+            val = val.strip().strip('"').strip("'")
+            env_list.append({"name": key, "default": val})
+    return env_list
+
+# -------------------------
+# Resolve the first "image:" line from docker-compose.yml
+# Supports ${VAR:-default} and ${VAR}
+# -------------------------
+VAR_RE = re.compile(r"\$\{([^:}]+)(?:[:-]([^}]+))?\}")
 
-# ---------------------------------------------------------
-# Parse image from docker-compose.yml using sample.env defaults
-# ---------------------------------------------------------
 def parse_image(compose_path: Path, env_vars: list) -> str:
-    if not compose_path.exists():
+    text = read_text_safe(compose_path)
+    if not text:
         return ""
 
     env_defaults = {v["name"]: v["default"] for v in env_vars}
 
-    for line in compose_path.read_text().splitlines():
+    for line in text.splitlines():
-        line = line.strip()
+        stripped = line.strip()
-        if line.startswith("image:"):
+        if stripped.startswith("image:"):
-            image = line.split("image:", 1)[1].strip().strip('"').strip("'")
+            image = stripped.split("image:", 1)[1].strip().strip('"').strip("'")
-
+            # replace ${VAR:-default} or ${VAR} using env_defaults
-            # Handle ${VAR:-default} or ${VAR}
+            def repl(m):
-            def replace_var(match):
+                var = m.group(1)
-                var_name = match.group(1)
+                default = m.group(2)
-                bash_default = match.group(2)
+                return env_defaults.get(var, default or "")
-                return env_defaults.get(var_name, bash_default if bash_default else "")
+            image = VAR_RE.sub(repl, image)
-
-            image = re.sub(r"\$\{([^:}]+)(?:[:-]([^}]+))?\}", replace_var, image)
             return image
-
     return ""
 
-# ---------------------------------------------------------
+# -------------------------
-# Parse ports from docker-compose.yml
+# Parse ports into a list of strings Portainer likes
-# Returns a list of dicts: {"container": port, "published": port, "protocol": "tcp"}
+# Examples supported:
-# ---------------------------------------------------------
+#   - "8080:80"
+#   - "9000"
+#   - "9000:9000/tcp"
+#   - "53:53/udp"
+# -------------------------
 def parse_ports(compose_path: Path):
-    if not compose_path.exists():
+    text = read_text_safe(compose_path)
+    if not text:
         return []
 
-    ports_list = []
+    ports = []
-    lines = compose_path.read_text().splitlines()
+    lines = text.splitlines()
     in_ports = False
-
     for line in lines:
         stripped = line.strip()
         if stripped.startswith("ports:"):
@@ -104,54 +183,34 @@ def parse_ports(compose_path: Path):
         if in_ports:
             if not stripped or not stripped.startswith("-"):
                 break
-
+            # get port string, strip comments/quotes
             port_str = stripped[1:].strip().split("#", 1)[0].strip().strip('"').strip("'")
             if not port_str:
                 continue
+            # normalize space
+            port_str = port_str.replace(" ", "")
+            # handle protocol suffix; keep as-is for Portainer
+            ports.append(port_str)
+    return ports
 
-            protocol = "tcp"
+# -------------------------
-            if "/" in port_str:
+# Parse volumes:
-                port_part, protocol_part = port_str.split("/", 1)
+# - Accept mapping forms:
-                protocol = protocol_part.strip()
+#     host:container[:ro|rw]
-            else:
+#     container[:ro|rw]
-                port_part = port_str
+#     named-volume:container[:ro|rw]
-
+# - For Portainer templates we'll provide:
-            if ":" in port_part:
+#     {"container": "/path/in/container", "bind": "<named-volume>"}
-                host_port, container_port = port_part.split(":", 1)
+#   where <named-volume> is generated as "<stack>-<container-path-sanitized>"
-            else:
+# -------------------------
-                container_port = port_part
-                host_port = container_port
-
-            try:
-                container_port = int(container_port.strip())
-                host_port = int(host_port.strip())
-            except ValueError:
-                continue
-
-            ports_list.append({
-                "container": container_port,
-                "published": host_port,
-                "protocol": protocol
-            })
-
-    return ports_list
-
-# ---------------------------------------------------------
-# Parse volumes from docker-compose.yml
-# Returns a list of dicts: {"name": "unique-name", "container": "/container/path"}
-# ---------------------------------------------------------
 def parse_volumes(compose_path: Path, stack_name: str):
-    """
+    text = read_text_safe(compose_path)
-    Parse volumes from docker-compose.yml and generate Portainer-ready entries.
+    if not text:
-    Host paths and VOL_PATH references are ignored; only container paths are used.
-    """
-    if not compose_path.exists():
         return []
 
-    volumes = []
+    vols = []
-    lines = compose_path.read_text().splitlines()
+    lines = text.splitlines()
     in_volumes = False
-
     for line in lines:
         stripped = line.strip()
         if stripped.startswith("volumes:"):
@@ -160,102 +219,94 @@ def parse_volumes(compose_path: Path, stack_name: str):
         if in_volumes:
             if not stripped or not stripped.startswith("-"):
                 break
-
+            vol_line = stripped[1:].strip().split("#", 1)[0].strip().strip('"').strip("'")
-            # Strip "- ", quotes, inline comments
+            if not vol_line:
-            volume_str = stripped[1:].strip().split("#", 1)[0].strip().strip('"').strip("'")
-            if not volume_str:
                 continue
-
+            # split from right to handle host:container:mode (we want container and ignore host)
-            # Split host:container[:ro|rw]
+            parts = vol_line.rsplit(":", 2)  # at most 3 parts
-            parts = volume_str.split(":")
+            # container part is last if there are 2 or more parts, else first
             if len(parts) == 1:
                 container_path = parts[0].strip()
             else:
-                container_path = parts[-1].strip()  # always take the last part as container path
+                container_path = parts[-2].strip() if len(parts) == 3 else parts[-1].strip()
-
+                # explanation:
-            # Remove :ro or :rw suffix if present
+                # - "host:container" -> parts len 2 -> parts[-1] == container
+                # - "host:container:ro" -> parts len 3 -> parts[-2] == container
+            # final strip of mode suffix if any left
             container_path = re.sub(r":(ro|rw)$", "", container_path)
+            # ensure container path starts with /
+            if not container_path.startswith("/"):
+                # if it's like "config.yaml" or similar, prefix with /
+                container_path = "/" + container_path.lstrip("/")
 
-            # Generate unique name for Portainer from stack + container path
+            # generate a safe named volume
-            name = f"{stack_name}-{container_path.strip('/').replace('/', '-')}"
+            safe = container_path.strip("/").replace("/", "-").replace(".", "-")
-            if not name:
+            if not safe:
-                name = f"{stack_name}-volume"
+                safe = "data"
+            vol_name = f"{stack_name}-{safe}"
+            vols.append({"container": container_path, "bind": vol_name})
+    return vols
 
-            volumes.append({"name": name, "container": container_path})
+# -------------------------
-
+# Build a Portainer template object for a project directory
-    return volumes
+# -------------------------
-
-# ---------------------------------------------------------
-# Parse environment variables from sample.env or env-sample
-# Returns a list of dicts: {"name": ..., "default": ...}
-# ---------------------------------------------------------
-def parse_env_vars(dir_path: Path):
-    env_file = None
-    if (dir_path / "sample.env").exists():
-        env_file = dir_path / "sample.env"
-    elif (dir_path / "env-sample").exists():
-        env_file = dir_path / "env-sample"
-
-    if not env_file:
-        return []
-
-    env_list = []
-    for line in env_file.read_text().splitlines():
-        line = line.strip()
-        if not line or line.startswith("#"):
-            continue
-        if "=" in line:
-            key, value = line.split("=", 1)
-            key = key.strip()
-            value = value.split("#", 1)[0].strip().strip('"').strip("'")
-            if key:
-                env_list.append({"name": key, "default": value})
-
-    return env_list
-
-# ---------------------------------------------------------
-# Build template object for a stack directory
-# ---------------------------------------------------------
 def generate_template_object(dir_path: Path, nologo: bool = False):
     name = dir_path.name
     compose_file = dir_path / "docker-compose.yml"
     readme_file = dir_path / "README.md"
 
     description = extract_overview(readme_file, name)
-    logo = find_logo_url(dir_path, nologo=False)
+    logo = find_logo_url(dir_path, nologo=nologo)
     env_vars = parse_env_vars(dir_path)
     image = parse_image(compose_file, env_vars)
     ports = parse_ports(compose_file)
     volumes = parse_volumes(compose_file, name)
 
-    env_entries = [
+    # env entries: only include those with a name
-        {"name": v["name"], "label": v["name"], "default": v["default"]}
+    env_entries = []
-        for v in env_vars if v["default"] is not None
+    for v in env_vars:
-    ]
+        if not v.get("name"):
+            continue
+        # leave default as empty string if missing (Portainer accepts empty)
+        env_entries.append({"name": v["name"], "label": v["name"], "default": v.get("default", "")})
 
-    return {
+    # Build stackfile raw URL on master branch
-        "type": 1,
+    stackfile_url = f"{GITHUB_RAW_BASE}/{name}/docker-compose.yml"
+
+    tpl = {
+        "type": "docker-compose",
         "title": name,
-        "description": description,
+        "name": name,
         "note": "Auto-generated template from repository",
         "categories": ["Auto"],
         "platform": "linux",
         "logo": logo,
+        "description": description,
+        # optional convenience field: image
         "image": image,
-        "repository": {
+        # Portainer expects a full stackfile URL
-            "url": "https://github.com/nickyeoman/docker-compose-cookbooks",
+        "stackfile": stackfile_url,
-            "stackfile": f"{name}/docker-compose.yml",
-            "stackfile_plain": False
-        },
         "env": env_entries,
-        "ports": ports,
-        "volumes": volumes
     }
 
-# ---------------------------------------------------------
+    # ports: Portainer accepts list of strings like "8000:80" or "53:53/udp"
-# Main script logic
+    if ports:
-# ---------------------------------------------------------
+        tpl["ports"] = ports
+
+    # volumes: produce list of objects with container and bind (named volume)
+    if volumes:
+        tpl["volumes"] = volumes
+
+    return tpl
+
+# -------------------------
+# Main
+# -------------------------
 def main():
+    parser = argparse.ArgumentParser(description="Generate Portainer templates.json from compose cookbooks")
+    parser.add_argument("--nologo", action="store_true", help="skip HEAD requests for logos (faster, dev)")
+    args = parser.parse_args()
+
     templates = []
 
     for item in REPO_ROOT.iterdir():
@@ -270,27 +321,20 @@ def main():
         if not compose_path.exists():
             continue
 
-        # ---------------------------------------------
+        # skip projects still using VOL_PATH (developer needs to fix)
-        # Skip stacks still using ${VOL_PATH
+        if SKIP_PAT in read_text_safe(compose_path):
-        # ---------------------------------------------
+            print(f"Skipping {item.name}: still uses ${{VOL_PATH}}")
-        compose_text = compose_path.read_text()
-        if "${VOL_PATH" in compose_text:
             continue
 
-        # ---------------------------------------------
+        tpl = generate_template_object(item, nologo=args.nologo)
-        # Generate Portainer template object
+        templates.append(tpl)
-        # ---------------------------------------------
-        templates.append(generate_template_object(item, nologo=True))
-
-    # Write output file
-    output_data = {
-        "version": "3",
-        "templates": templates
-    }
-
-    OUTPUT_FILE.write_text(json.dumps(output_data, indent=2))
-    print(f"Generated {OUTPUT_FILE} with templates for Docker Compose stacks.")
 
+    output = {"version": "3", "templates": templates}
+    # Write atomically if possible
+    tmp = OUTPUT_FILE.with_suffix(".tmp")
+    tmp.write_text(json.dumps(output, indent=2))
+    tmp.replace(OUTPUT_FILE)
+    print(f"Generated {OUTPUT_FILE} with {len(templates)} templates.")
 
 if __name__ == "__main__":
     main()