Merge pull request #704 from pulp/update-ci/main

Update CI files for branch main

.ci/ansible/Containerfile.j2

@@ -15,6 +15,9 @@ RUN pip3 install
 {%- endif -%}
 {%- for item in plugins -%}
 {{ " " }}{{ item.source }}
+{%- if item.upperbounds | default(false) -%}
+{{ " " }}-c ./{{ item.name }}/upperbounds_constraints.txt
+{%- endif -%}
 {%- if item.lowerbounds | default(false) -%}
 {{ " " }}-c ./{{ item.name }}/lowerbounds_constraints.txt
 {%- endif -%}

.ci/scripts/calc_constraints.py

@@ -0,0 +1,116 @@
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --github pulp_python' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
+import argparse
+import fileinput
+import urllib.request
+import sys
+from packaging.requirements import Requirement
+from packaging.version import Version
+import yaml
+
+
+CORE_TEMPLATE_URL = "https://raw.githubusercontent.com/pulp/pulpcore/main/template_config.yml"
+
+
+def fetch_pulpcore_upper_bound(requirement):
+    with urllib.request.urlopen(CORE_TEMPLATE_URL) as f:
+        template = yaml.safe_load(f.read())
+    supported_versions = template["supported_release_branches"]
+    supported_versions.append(template["latest_release_branch"])
+    applicable_versions = sorted(
+        requirement.specifier.filter((Version(v) for v in supported_versions))
+    )
+    if len(applicable_versions) == 0:
+        raise Exception("No supported pulpcore version in required range.")
+    return f"{requirement.name}~={applicable_versions[-1]}"
+
+
+def split_comment(line):
+    split_line = line.split("#", maxsplit=1)
+    try:
+        comment = "  # " + split_line[1].strip()
+    except IndexError:
+        comment = ""
+    return split_line[0].strip(), comment
+
+
+def to_upper_bound(req):
+    try:
+        requirement = Requirement(req)
+    except ValueError:
+        return f"# UNPARSABLE: {req}"
+    else:
+        if requirement.name == "pulpcore":
+            # An exception to allow for pulpcore deprecation policy.
+            return fetch_pulpcore_upper_bound(requirement)
+        for spec in requirement.specifier:
+            if spec.operator == "~=":
+                return f"# NO BETTER CONSTRAINT: {req}"
+            if spec.operator == "<=":
+                operator = "=="
+                max_version = spec.version
+                return f"{requirement.name}{operator}{max_version}"
+            if spec.operator == "<":
+                operator = "~="
+                version = Version(spec.version)
+                if version.micro != 0:
+                    max_version = f"{version.major}.{version.minor}.{version.micro-1}"
+                elif version.minor != 0:
+                    max_version = f"{version.major}.{version.minor-1}"
+                else:
+                    return f"# NO BETTER CONSTRAINT: {req}"
+                return f"{requirement.name}{operator}{max_version}"
+        return f"# NO UPPER BOUND: {req}"
+
+
+def to_lower_bound(req):
+    try:
+        requirement = Requirement(req)
+    except ValueError:
+        return f"# UNPARSABLE: {req}"
+    else:
+        for spec in requirement.specifier:
+            if spec.operator == ">=":
+                if requirement.name == "pulpcore":
+                    # Currently an exception to allow for pulpcore bugfix releases.
+                    # TODO Semver libraries should be allowed too.
+                    operator = "~="
+                else:
+                    operator = "=="
+                min_version = spec.version
+                return f"{requirement.name}{operator}{min_version}"
+        return f"# NO LOWER BOUND: {req}"
+
+
+def main():
+    """Calculate constraints for the lower bound of dependencies where possible."""
+    parser = argparse.ArgumentParser(
+        prog=sys.argv[0],
+        description="Calculate constraints for the lower or upper bound of dependencies where "
+        "possible.",
+    )
+    parser.add_argument("-u", "--upper", action="store_true")
+    parser.add_argument("filename", nargs="*")
+    args = parser.parse_args()
+
+    with fileinput.input(files=args.filename) as req_file:
+        for line in req_file:
+            if line.strip().startswith("#"):
+                # Shortcut comment only lines
+                print(line.strip())
+            else:
+                req, comment = split_comment(line)
+                if args.upper:
+                    new_req = to_upper_bound(req)
+                else:
+                    new_req = to_lower_bound(req)
+                print(new_req + comment)
+
+
+if __name__ == "__main__":
+    main()

.github/template_gitref

@@ -1 +1 @@
-2021.08.26-349-gba81617
+2021.08.26-354-g82d22de

.github/workflows/nightly.yml

@@ -65,55 +65,3 @@ jobs:
           branch: "changelog/update"
           delete-branch: true
           path: "pulp_python"
-
-  publish:
-    runs-on: ubuntu-latest
-    needs: test
-
-    steps:
-      - uses: "actions/checkout@v4"
-        with:
-          fetch-depth: 1
-          path: "pulp_python"
-
-      - uses: actions/download-artifact@v4
-        with:
-          name: "plugin_package"
-          path: "pulp_python/dist/"
-
-      - uses: "actions/setup-python@v5"
-        with:
-          python-version: "3.11"
-
-      - name: "Install python dependencies"
-        run: |
-          echo ::group::PYDEPS
-          pip install requests 'packaging~=21.3' mkdocs pymdown-extensions 'Jinja2<3.1'
-          echo ::endgroup::
-
-      - name: "Set environment variables"
-        run: |
-          echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV
-
-      - name: Download built docs
-        uses: actions/download-artifact@v4
-        with:
-          name: "docs.tar"
-          path: "pulp_python"
-
-      - name: Download Python client docs
-        uses: actions/download-artifact@v4
-        with:
-          name: "python-client-docs.tar"
-          path: "pulp_python"
-
-      - name: "Setting secrets"
-        run: |
-          python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT"
-        env:
-          SECRETS_CONTEXT: "${{ toJson(secrets) }}"
-
-      - name: Publish docs to pulpproject.org
-        run: |
-          tar -xvf docs.tar -C ./docs
-          .github/workflows/scripts/publish_docs.sh nightly ${GITHUB_REF##*/}

.github/workflows/publish.yml

@@ -137,52 +137,6 @@ jobs:
       - name: "Publish client to rubygems"
         run: |
           bash .github/workflows/scripts/publish_client_gem.sh ${{ github.ref_name }}
-  publish-docs:
-    runs-on: "ubuntu-latest"
-    needs:
-      - "build"
-
-    env:
-      GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
-
-    steps:
-      - uses: "actions/checkout@v4"
-        with:
-          fetch-depth: 1
-          path: "pulp_python"
-
-      - uses: "actions/setup-python@v5"
-        with:
-          python-version: "3.11"
-
-      - name: "Install python dependencies"
-        run: |
-          echo ::group::PYDEPS
-          pip install 'packaging~=21.3' requests
-          echo ::endgroup::
-
-      - name: "Setting secrets"
-        run: |
-          python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT"
-        env:
-          SECRETS_CONTEXT: "${{ toJson(secrets) }}"
-
-      - name: "Download built docs"
-        uses: "actions/download-artifact@v4"
-        with:
-          name: "docs.tar"
-          path: "pulp_python/"
-
-      - name: "Download Python client docs"
-        uses: "actions/download-artifact@v4"
-        with:
-          name: "python-client-docs.tar"
-          path: "pulp_python/"
-
-      - name: "Publish docs to pulpproject.org"
-        run: |
-          tar -xvf docs.tar
-          .github/workflows/scripts/publish_docs.sh tag ${{ github.ref_name }}
 
   create-gh-release:
     runs-on: "ubuntu-latest"
@@ -191,7 +145,6 @@ jobs:
       - "publish-package"
       - "publish-python-bindings"
       - "publish-ruby-bindings"
-      - "publish-docs"
 
     steps:
       - name: "Create release on GitHub"

.github/workflows/scripts/before_install.sh

@@ -65,9 +65,11 @@ then
   exit $s
 fi
 
+if [[ "$TEST" = "pulp" ]]; then
+  python3 .ci/scripts/calc_constraints.py -u requirements.txt > upperbounds_constraints.txt
+fi
 if [[ "$TEST" = "lowerbounds" ]]; then
-  python3 .ci/scripts/calc_deps_lowerbounds.py > lowerbounds_constraints.txt
-  sed -i 's/\[.*\]//g' lowerbounds_constraints.txt
+  python3 .ci/scripts/calc_constraints.py requirements.txt > lowerbounds_constraints.txt
 fi
 
 if [ -f $POST_BEFORE_INSTALL ]; then