Merge pull request #2698 from nf-core/dev

dev -> master for 2.12

.editorconfig

@@ -10,3 +10,13 @@ indent_style = space
 
 [*.{md,yml,yaml,html,css,scss,js,cff}]
 indent_size = 2
+
+# ignore python and markdown files
+[*.py]
+indent_style = unset
+
+[**/{CONTRIBUTING,README}.md]
+indent_style = unset
+
+[**/Makefile]
+indent_style = unset

.github/.coveragerc

@@ -1,2 +1,5 @@
 [run]
-omit = nf_core/pipeline-template/*
+omit = nf_core/*-template/*
+source = nf_core
+relative_files = True
+

.github/CONTRIBUTING.md

@@ -35,45 +35,28 @@ pip install -e .
 
 ## Code formatting
 
-### Black
+### Ruff
 
-All Python code in nf-core/tools must be passed through the [Black Python code formatter](https://black.readthedocs.io/en/stable/).
+All Python code in nf-core/tools must be passed through the [Ruff code linter and formatter](https://github.com/astral-sh/ruff).
 This ensures a harmonised code formatting style throughout the package, from all contributors.
 
-You can run Black on the command line (it's included in `requirements-dev.txt`) - eg. to run recursively on the whole repository:
+You can run Ruff on the command line (it's included in `requirements-dev.txt`) - eg. to run recursively on the whole repository:
 
 ```bash
-black .
+ruff format .
 ```
 
-Alternatively, Black has [integrations for most common editors](https://black.readthedocs.io/en/stable/editor_integration.html)
+Alternatively, Ruff has [integrations for most common editors](https://github.com/astral-sh/ruff-lsp) and VSCode(https://github.com/astral-sh/ruff-vscode)
 to automatically format code when you hit save.
-You can also set it up to run when you [make a commit](https://black.readthedocs.io/en/stable/version_control_integration.html).
 
 There is an automated CI check that runs when you open a pull-request to nf-core/tools that will fail if
-any code does not adhere to Black formatting.
+any code does not adhere to Ruff formatting.
 
-### isort
-
-All Python code must also be passed through [isort](https://pycqa.github.io/isort/index.html).
-This ensures a harmonised imports throughout the package, from all contributors.
-
-To run isort on the command line recursively on the whole repository you can use:
-
-```bash
-isort .
-```
-
-isort also has [plugins for most common editors](https://github.com/pycqa/isort/wiki/isort-Plugins)
-to automatically format code when you hit save.
-Or [version control integration](https://pycqa.github.io/isort/docs/configuration/pre-commit.html) to set it up to run when you make a commit.
-
-There is an automated CI check that runs when you open a pull-request to nf-core/tools that will fail if
-any code does not adhere to isort formatting.
+Ruff has been adopted for linting and formatting in replacement of Black, isort (for imports) and pyupgrade. It also includes Flake8.
 
 ### pre-commit hooks
 
-This repository comes with [pre-commit](https://pre-commit.com/) hooks for black, isort and Prettier. pre-commit automatically runs checks before a commit is committed into the git history. If all checks pass, the commit is made, if files are changed by the pre-commit hooks, the user is informed and has to stage the changes and attempt the commit again.
+This repository comes with [pre-commit](https://pre-commit.com/) hooks for ruff and Prettier. pre-commit automatically runs checks before a commit is committed into the git history. If all checks pass, the commit is made, if files are changed by the pre-commit hooks, the user is informed and has to stage the changes and attempt the commit again.
 
 You can use the pre-commit hooks if you like, but you don't have to. The CI on Github will run the same checks as the tools installed with pre-commit. If the pre-commit checks pass, then the same checks in the CI will pass, too.
 

.github/renovate.json5

@@ -1,5 +1,17 @@
 {
     $schema: "https://docs.renovatebot.com/renovate-schema.json",
     extends: ["github>nf-core/ops//.github/renovate/default.json5"],
+    ignorePaths: ["**/nf_core/pipeline-template/modules/nf-core/**"],
     baseBranches: ["dev"],
+    packageRules: [
+        {
+            matchDatasources: ["docker"],
+            matchPackageNames: ["python"],
+            versioning: "pep440",
+        },
+        {
+            matchDatasources: ["docker"],
+            registryUrls: ["docker.io"],
+        },
+    ],
 }

.github/workflows/branch.yml

@@ -18,7 +18,7 @@ jobs:
       # If the above check failed, post a comment on the PR explaining the failure
       - name: Post PR comment
         if: failure()
-        uses: mshick/add-pr-comment@v1
+        uses: mshick/add-pr-comment@v2
         with:
           message: |
             ## This PR is against the `master` branch :x:

.github/workflows/changelog.py

@@ -0,0 +1,228 @@
+"""
+Taken from https://github.com/MultiQC/MultiQC/blob/main/.github/workflows/changelog.py and updated for nf-core
+
+To be called by a CI action. Assumes the following environment variables are set:
+PR_TITLE, PR_NUMBER, GITHUB_WORKSPACE.
+
+Adds a line into the CHANGELOG.md:
+* Looks for the section to add the line to, based on the PR title, e.g. `Template:`, `Modules:`.
+* All other change will go under the "### General" section.
+* If an entry for the PR is already added, it will not run.
+
+Other assumptions:
+- CHANGELOG.md has a running section for an ongoing "dev" version
+(i.e. titled "## nf-core vX.Ydev").
+"""
+
+import os
+import re
+import sys
+from pathlib import Path
+from typing import List
+
+REPO_URL = "https://github.com/nf-core/tools"
+
+# Assumes the environment is set by the GitHub action.
+pr_title = os.environ["PR_TITLE"]
+pr_number = os.environ["PR_NUMBER"]
+comment = os.environ.get("COMMENT", "")
+workspace_path = Path(os.environ.get("GITHUB_WORKSPACE", ""))
+
+assert pr_title, pr_title
+assert pr_number, pr_number
+
+# Trim the PR number added when GitHub squashes commits, e.g. "Template: Updated (#2026)"
+pr_title = pr_title.removesuffix(f" (#{pr_number})")
+
+changelog_path = workspace_path / "CHANGELOG.md"
+
+if any(
+    line in pr_title.lower()
+    for line in [
+        "skip changelog",
+        "skip change log",
+        "no changelog",
+        "no change log",
+        "bump version",
+    ]
+):
+    print("Skipping changelog update")
+    sys.exit(0)
+
+
+def _determine_change_type(pr_title) -> tuple[str, str]:
+    """
+    Determine the type of the PR: Template, Download, Linting, Modules, Subworkflows, or General
+    Returns a tuple of the section name and the module info.
+    """
+    sections = {
+        "Template": "### Template",
+        "Download": "### Download",
+        "Linting": "### Linting",
+        "Modules": "### Modules",
+        "Subworkflows": "### Subworkflows",
+    }
+    current_section_header = "### General"
+    current_section = "General"
+
+    # Check if the PR in any of the sections.
+    for section, section_header in sections.items():
+        # check if the PR title contains any of the section headers, with some loose matching, e.g. removing plural and suffixes
+        if re.sub(r"s$", "", section.lower().replace("ing", "")) in pr_title.lower():
+            current_section_header = section_header
+            current_section = section
+    print(f"Detected section: {current_section}")
+    return current_section, current_section_header
+
+
+# Determine the type of the PR
+section, section_header = _determine_change_type(pr_title)
+
+# Remove section indicator from the PR title.
+pr_title = re.sub(rf"{section}[:\s]*", "", pr_title, flags=re.IGNORECASE)
+
+# Prepare the change log entry.
+pr_link = f"([#{pr_number}]({REPO_URL}/pull/{pr_number}))"
+
+# Handle manual changelog entries through comments.
+if comment := comment.removeprefix("@nf-core-bot changelog").strip():
+    print(f"Adding manual changelog entry: {comment}")
+    pr_title = comment
+new_lines = [
+    f"- {pr_title} {pr_link}\n",
+]
+
+print(f"Adding new lines into section '{section}':\n" + "".join(new_lines))
+
+# Finally, updating the changelog.
+# Read the current changelog lines. We will print them back as is, except for one new
+# entry, corresponding to this new PR.
+with changelog_path.open("r") as f:
+    orig_lines = f.readlines()
+updated_lines: List[str] = []
+
+
+def _skip_existing_entry_for_this_pr(line: str, same_section: bool = True) -> str:
+    if line.strip().endswith(pr_link):
+        print(f"Found existing entry for this pull request #{pr_number}:")
+        existing_lines = [line]
+        if new_lines and new_lines == existing_lines and same_section:
+            print(f"Found existing identical entry for this pull request #{pr_number} in the same section:")
+            print("".join(existing_lines))
+            sys.exit(0)  # Just leaving the CHANGELOG intact
+        else:
+            print(
+                f"Found existing entry for this pull request #{pr_number}. It will be replaced and/or moved to proper section"
+            )
+            print("".join(existing_lines))
+            for _ in range(len(existing_lines)):
+                try:
+                    line = orig_lines.pop(0)
+                except IndexError:
+                    break
+    return line
+
+
+# Find the next line in the change log that matches the pattern "# nf-core/tools v.*dev"
+# If it doesn't exist, exist with code 1 (let's assume that a new section is added
+# manually or by CI when a release is pushed).
+# Else, find the next line that matches the `section` variable, and insert a new line
+# under it (we also assume that section headers are added already).
+inside_version_dev = False
+already_added_entry = False
+while orig_lines:
+    line = orig_lines.pop(0)
+
+    # If the line already contains a link to the PR, don't add it again.
+    line = _skip_existing_entry_for_this_pr(line, same_section=False)
+
+    if line.startswith("# ") and not line.strip() == "# nf-core/tools: Changelog":  # Version header, e.g. "# v2.12dev"
+        print(f"Found version header: {line.strip()}")
+        updated_lines.append(line)
+
+        # Parse version from the line `# v2.12dev` or
+        # `# [v2.11.1 - Magnesium Dragon Patch](https://github.com/nf-core/tools/releases/tag/2.11) - [2023-12-20]` ...
+        if not (m := re.match(r".*(v\d+\.\d+(dev)?).*", line)):
+            print(f"Cannot parse version from line {line.strip()}.", file=sys.stderr)
+            sys.exit(1)
+        version = m.group(1)
+
+        if not inside_version_dev:
+            if not version.endswith("dev"):
+                print(
+                    "Can't find a 'dev' version section in the changelog. Make sure "
+                    "it's created, and all the required sections, e.g. `### Template` are created under it .",
+                    file=sys.stderr,
+                )
+                sys.exit(1)
+            inside_version_dev = True
+        else:
+            if version.endswith("dev"):
+                print(
+                    f"Found another 'dev' version section in the changelog, make"
+                    f"sure to change it to a 'release' stable version tag. "
+                    f"Line: {line.strip()}",
+                    file=sys.stderr,
+                )
+                sys.exit(1)
+            # We are past the dev version, so just add back the rest of the lines and break.
+            while orig_lines:
+                line = orig_lines.pop(0)
+                line = _skip_existing_entry_for_this_pr(line, same_section=False)
+                if line:
+                    updated_lines.append(line)
+            break
+        continue
+    print(f"Found line: {line.strip()}")
+    print(f"inside_version_dev: {inside_version_dev}")
+    print(f"section_header: {section_header}")
+    if inside_version_dev and line.lower().startswith(section_header.lower()):  # Section of interest header
+        print(f"Found section header: {line.strip()}")
+        if already_added_entry:
+            print(
+                f"Already added new lines into section {section}, is the section duplicated?",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+        updated_lines.append(line)
+        # Collecting lines until the next section.
+        section_lines: List[str] = []
+        while True:
+            line = orig_lines.pop(0)
+            if line.startswith("#"):
+                print(f"Found the next section header: {line.strip()}")
+                # Found the next section header, so need to put all the lines we collected.
+                updated_lines.append("\n")
+                _updated_lines = [_l for _l in section_lines + new_lines if _l.strip()]
+                updated_lines.extend(_updated_lines)
+                updated_lines.append("\n")
+                if new_lines:
+                    print(f"Updated {changelog_path} section '{section}' with lines:\n" + "".join(new_lines))
+                else:
+                    print(f"Removed existing entry from {changelog_path} section '{section}'")
+                already_added_entry = True
+                # Pushing back the next section header line
+                orig_lines.insert(0, line)
+                break
+            # If the line already contains a link to the PR, don't add it again.
+            line = _skip_existing_entry_for_this_pr(line, same_section=True)
+            section_lines.append(line)
+    else:
+        updated_lines.append(line)
+
+
+def collapse_newlines(lines: List[str]) -> List[str]:
+    updated = []
+    for idx in range(len(lines)):
+        if idx != 0 and not lines[idx].strip() and not lines[idx - 1].strip():
+            continue
+        updated.append(lines[idx])
+    return updated
+
+
+updated_lines = collapse_newlines(updated_lines)
+
+
+# Finally, writing the updated lines back.
+with changelog_path.open("w") as f:
+    f.writelines(updated_lines)