nf-core · skrakau · Aug 13, 2020 · Jul 30, 2020 · Jul 31, 2020
@@ -0,0 +1,5 @@
+# Dockstore config version, not pipeline version
+version: 1.2
+workflows:
+  - subclass: nfl
+    primaryDescriptorPath: /nextflow.config
@@ -46,12 +46,12 @@ These tests are run both with the latest available version of `Nextflow` and als
 
 ## Patch
 
-: warning: Only in the unlikely and regretful event of a release happening with a bug.
+:warning: Only in the unlikely and regretful event of a release happening with a bug.
 
 * On your own fork, make a new branch `patch` based on `upstream/master`.
 * Fix the bug, and bump version (X.Y.Z+1).
 * A PR should be made on `master` from patch to directly this particular bug.
 
 ## Getting help
 
-For further information/help, please consult the [nf-core/mag documentation](https://nf-co.re/nf-core/mag/docs) and don't hesitate to get in touch on the nf-core Slack [#mag](https://nfcore.slack.com/channels/mag) channel ([join our Slack here](https://nf-co.re/join/slack)).
+For further information/help, please consult the [nf-core/mag documentation](https://nf-co.re/mag/docs) and don't hesitate to get in touch on the nf-core Slack [#mag](https://nfcore.slack.com/channels/mag) channel ([join our Slack here](https://nf-co.re/join/slack)).
@@ -1,24 +1,27 @@
+<!--
 # nf-core/mag bug report
 
 Hi there!
 
 Thanks for telling us about a problem with the pipeline.
+
 Please delete this text and anything that's not relevant from the template below:
+-->
 
-## Describe the bug
+## Description of the bug
 
-A clear and concise description of what the bug is.
+<!-- A clear and concise description of what the bug is. -->
 
 ## Steps to reproduce
 
 Steps to reproduce the behaviour:
 
-1. Command line: `nextflow run ...`
-2. See error: _Please provide your error message_
+1. Command line: <!-- [e.g. `nextflow run ...`] -->
+2. See error: <!-- [Please provide your error message] -->
 
 ## Expected behaviour
 
-A clear and concise description of what you expected to happen.
+<!-- A clear and concise description of what you expected to happen. -->
 
 ## System
 
@@ -39,4 +42,4 @@ A clear and concise description of what you expected to happen.
 
 ## Additional context
 
-Add any other context about the problem here.
+<!-- Add any other context about the problem here. -->
@@ -1,24 +1,26 @@
+<!--
 # nf-core/mag feature request
 
 Hi there!
 
 Thanks for suggesting a new feature for the pipeline!
 Please delete this text and anything that's not relevant from the template below:
+-->
 
 ## Is your feature request related to a problem? Please describe
 
-A clear and concise description of what the problem is.
+<!-- A clear and concise description of what the problem is. -->
 
-Ex. I'm always frustrated when [...]
+<!-- e.g. [I'm always frustrated when ...] -->
 
 ## Describe the solution you'd like
 
-A clear and concise description of what you want to happen.
+<!-- A clear and concise description of what you want to happen. -->
 
 ## Describe alternatives you've considered
 
-A clear and concise description of any alternative solutions or features you've considered.
+<!-- A clear and concise description of any alternative solutions or features you've considered. -->
 
 ## Additional context
 
-Add any other context about the feature request here.
+<!-- Add any other context about the feature request here. -->
@@ -1,19 +1,20 @@
+<!--
 # nf-core/mag pull request
 
 Many thanks for contributing to nf-core/mag!
 
 Please fill in the appropriate checklist below (delete whatever is not relevant).
 These are the most common things requested on pull requests (PRs).
 
+Remember that PRs should be made against the dev branch, unless you're preparing a pipeline release.
+
+Learn more about contributing: [CONTRIBUTING.md](https://github.com/nf-core/mag/tree/master/.github/CONTRIBUTING.md)
+-->
+
 ## PR checklist
 
 - [ ] This comment contains a description of changes (with reason)
+- [ ] `CHANGELOG.md` is updated
 - [ ] If you've fixed a bug or added code that should be tested, add tests!
-- [ ] If necessary, also make a PR on the [nf-core/mag branch on the nf-core/test-datasets repo](https://github.com/nf-core/test-datasets/pull/new/nf-core/mag)
-- [ ] Ensure the test suite passes (`nextflow run . -profile test,docker`).
-- [ ] Make sure your code lints (`nf-core lint .`).
 - [ ] Documentation in `docs` is updated
-- [ ] `CHANGELOG.md` is updated
-- [ ] `README.md` is updated
-
-**Learn more about contributing:** [CONTRIBUTING.md](https://github.com/nf-core/mag/tree/master/.github/CONTRIBUTING.md)
+- [ ] If necessary, also make a PR on the [nf-core/mag branch on the nf-core/test-datasets repo](https://github.com/nf-core/test-datasets/pull/new/nf-core/mag)
@@ -0,0 +1,40 @@
+name: nf-core AWS full size tests
+# This workflow is triggered on push to the master branch.
+# It runs the -profile 'test_full' on AWS batch
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  run-awstest:
+    name: Run AWS full tests
+    if: github.repository == 'nf-core/mag'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup Miniconda
+        uses: goanpeca/[email protected]
+        with:
+          auto-update-conda: true
+          python-version: 3.7
+      - name: Install awscli
+        run: conda install -c conda-forge awscli
+      - name: Start AWS batch job
+        # TODO nf-core: You can customise AWS full pipeline tests as required
+        # Add full size test data (but still relatively small datasets for few samples)
+        # on the `test_full.config` test runs with only one set of parameters
+        # Then specify `-profile test_full` instead of `-profile test` on the AWS batch command
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          TOWER_ACCESS_TOKEN: ${{ secrets.AWS_TOWER_TOKEN }}
+          AWS_JOB_DEFINITION: ${{ secrets.AWS_JOB_DEFINITION }}
+          AWS_JOB_QUEUE: ${{ secrets.AWS_JOB_QUEUE }}
+          AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
+        run: |
+          aws batch submit-job \
+            --region eu-west-1 \
+            --job-name nf-core-mag \
+            --job-queue $AWS_JOB_QUEUE \
+            --job-definition $AWS_JOB_DEFINITION \
+            --container-overrides '{"command": ["nf-core/mag", "-r '"${GITHUB_SHA}"' -profile test --outdir s3://'"${AWS_S3_BUCKET}"'/mag/results-'"${GITHUB_SHA}"' -w s3://'"${AWS_S3_BUCKET}"'/mag/work-'"${GITHUB_SHA}"' -with-tower"], "environment": [{"name": "TOWER_ACCESS_TOKEN", "value": "'"$TOWER_ACCESS_TOKEN"'"}]}'
@@ -0,0 +1,40 @@
+name: nf-core AWS test
+# This workflow is triggered on push to the master branch.
+# It runs the -profile 'test' on AWS batch
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  run-awstest:
+    name: Run AWS tests
+    if: github.repository == 'nf-core/mag'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup Miniconda
+        uses: goanpeca/[email protected]
+        with:
+          auto-update-conda: true
+          python-version: 3.7
+      - name: Install awscli
+        run: conda install -c conda-forge awscli
+      - name: Start AWS batch job
+        # TODO nf-core: You can customise CI pipeline run tests as required
+        # For example: adding multiple test runs with different parameters
+        # Remember that you can parallelise this by using strategy.matrix
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          TOWER_ACCESS_TOKEN: ${{ secrets.AWS_TOWER_TOKEN }}
+          AWS_JOB_DEFINITION: ${{ secrets.AWS_JOB_DEFINITION }}
+          AWS_JOB_QUEUE: ${{ secrets.AWS_JOB_QUEUE }}
+          AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
+        run: |
+          aws batch submit-job \
+          --region eu-west-1 \
+          --job-name nf-core-mag \
+          --job-queue $AWS_JOB_QUEUE \
+          --job-definition $AWS_JOB_DEFINITION \
+          --container-overrides '{"command": ["nf-core/mag", "-r '"${GITHUB_SHA}"' -profile test --outdir s3://'"${AWS_S3_BUCKET}"'/mag/results-'"${GITHUB_SHA}"' -w s3://'"${AWS_S3_BUCKET}"'/mag/work-'"${GITHUB_SHA}"' -with-tower"], "environment": [{"name": "TOWER_ACCESS_TOKEN", "value": "'"$TOWER_ACCESS_TOKEN"'"}]}'mag/work-'"${GITHUB_SHA}"' -with-tower"], "environment": [{"name": "TOWER_ACCESS_TOKEN", "value": "'"$TOWER_ACCESS_TOKEN"'"}]}'
@@ -3,14 +3,35 @@ name: nf-core branch protection
 # It fails when someone tries to make a PR against the nf-core `master` branch instead of `dev`
 on:
   pull_request:
-    branches:
-    - master
+    branches: [master]
 
 jobs:
   test:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     steps:
-      # PRs are only ok if coming from an nf-core `dev` branch or a fork `patch` branch
+      # PRs to the nf-core repo master branch are only ok if coming from the nf-core repo `dev` or any `patch` branches
       - name: Check PRs
+        if: github.repository == 'nf-core/mag'
         run: |
-          { [[ $(git remote get-url origin) == *nf-core/mag ]] && [[ ${GITHUB_HEAD_REF} = "dev" ]]; } || [[ ${GITHUB_HEAD_REF} == "patch" ]]
+          { [[ ${{github.event.pull_request.head.repo.full_name}} == nf-core/mag ]] && [[ $GITHUB_HEAD_REF = "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]]
+
+
+      # If the above check failed, post a comment on the PR explaining the failure
+      # NOTE - this doesn't currently work if the PR is coming from a fork, due to limitations in GitHub actions secrets
+      - name: Post PR comment
+        if: failure()
+        uses: mshick/add-pr-comment@v1
+        with:
+          message: |
+            Hi @${{ github.event.pull_request.user.login }},
+
+            It looks like this pull-request is has been made against the ${{github.event.pull_request.head.repo.full_name}} `master` branch.
+            The `master` branch on nf-core repositories should always contain code from the latest release.
+            Because of this, PRs to `master` are only allowed if they come from the ${{github.event.pull_request.head.repo.full_name}} `dev` branch.
+
+            You do not need to close this PR, you can change the target branch to `dev` by clicking the _"Edit"_ button at the top of this page.
+
+            Thanks again for your contribution!
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          allow-repeats: false
+
@@ -1,30 +1,55 @@
 name: nf-core CI
-# This workflow is triggered on pushes and PRs to the repository.
-# It runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
-on: [push, pull_request]
+# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
+on:
+  push:
+    branches:
+      - dev
+  pull_request:
+  release:
+    types: [published]
 
 jobs:
   test:
+    name: Run workflow tests
+    # Only run on push if this is the nf-core dev branch (merged PRs)
+    if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/mag') }}
+    runs-on: ubuntu-latest
     env:
       NXF_VER: ${{ matrix.nxf_ver }}
       NXF_ANSI_LOG: false
-    runs-on: ubuntu-latest
     strategy:
       matrix:
         # Nextflow versions: check pipeline minimum and current latest
         nxf_ver: ['19.10.0', '']
     steps:
-      - uses: actions/checkout@v2
-      - name: Install Nextflow
-        run: |
-          wget -qO- get.nextflow.io | bash
-          sudo mv nextflow /usr/local/bin/
+      - name: Check out pipeline code
+        uses: actions/checkout@v2
+
+      - name: Check if Dockerfile or Conda environment changed
+        uses: technote-space/get-diff-action@v1
+        with:
+          PREFIX_FILTER: |
+            Dockerfile
+            environment.yml
+
+      - name: Build new docker image
+        if: env.GIT_DIFF
+        run: docker build --no-cache . -t nfcore/mag:dev
+
       - name: Pull docker image
+        if: ${{ !env.GIT_DIFF }}
         run: |
           docker pull nfcore/mag:dev
           docker tag nfcore/mag:dev nfcore/mag:dev
+
+      - name: Install Nextflow
+        run: |
+          wget -qO- get.nextflow.io | bash
+          sudo mv nextflow /usr/local/bin/
+
       - name: Run pipeline with test data
+        # TODO nf-core: You can customise CI pipeline run tests as required
+        # For example: adding multiple test runs with different parameters
+        # Remember that you can parallelise this by using strategy.matrix
         run: |
-          # TODO nf-core: You can customise CI pipeline run tests as required
-          # (eg. adding multiple test runs with different parameters)
           nextflow run ${GITHUB_WORKSPACE} -profile test,docker
@@ -33,18 +33,36 @@ jobs:
   nf-core:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+
+      - name: Check out pipeline code
+        uses: actions/checkout@v2
+
       - name: Install Nextflow
         run: |
           wget -qO- get.nextflow.io | bash
           sudo mv nextflow /usr/local/bin/
+
       - uses: actions/setup-python@v1
         with:
           python-version: '3.6'
           architecture: 'x64'
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install nf-core
+
       - name: Run nf-core lint
-        run: nf-core lint ${GITHUB_WORKSPACE}
+        env:
+          GITHUB_COMMENTS_URL: ${{ github.event.pull_request.comments_url }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_PR_COMMIT: ${{ github.event.pull_request.head.sha }}
+        run: nf-core -l lint_log.txt lint ${GITHUB_WORKSPACE}
+
+      - name: Upload linting log file artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: linting-log-file
+          path: lint_log.txt
+
@@ -0,0 +1,40 @@
+name: nf-core Docker push
+# This builds the docker image and pushes it to DockerHub
+# Runs on nf-core repo releases and push event to 'dev' branch (PR merges)
+on:
+  push:
+    branches:
+      - dev
+  release:
+    types: [published]
+
+jobs:
+  push_dockerhub:
+    name: Push new Docker image to Docker Hub
+    runs-on: ubuntu-latest
+    # Only run for the nf-core repo, for releases and merged PRs
+    if: ${{ github.repository == 'nf-core/mag' }}
+    env:
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      DOCKERHUB_PASS: ${{ secrets.DOCKERHUB_PASS }}
+    steps:
+      - name: Check out pipeline code
+        uses: actions/checkout@v2
+
+      - name: Build new docker image
+        run: docker build --no-cache . -t nfcore/mag:latest
+
+      - name: Push Docker image to DockerHub (dev)
+        if: ${{ github.event_name == 'push' }}
+        run: |
+          echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin
+          docker tag nfcore/mag:latest nfcore/mag:dev
+          docker push nfcore/mag:dev
+
+      - name: Push Docker image to DockerHub (release)
+        if: ${{ github.event_name == 'release' }}
+        run: |
+          echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin
+          docker push nfcore/mag:latest
+          docker tag nfcore/mag:latest nfcore/mag:${{ github.event.release.tag_name }}
+          docker push nfcore/mag:${{ github.event.release.tag_name }}
@@ -5,4 +5,5 @@ results/
 .DS_Store
 tests/
 testing/
+testing*
 *.pyc