Target app mayhem security (#128)

* Use MAYHEM_URL secret rather than hard-coded beta instance in workflow

* Set owner

* Add fail on defects test

* Wait if fail-on-defects set

* Restructure __tests__

* Allow overriding package path

Author: mjkoo

.github/workflows/main.yml

@@ -1,14 +1,15 @@
-name: 'build-test'
+name: "build-test"
 
 on:
   push:
-    branches: [ main ]
+    branches: [main]
   pull_request:
-    branches: [ main ]
+    branches: [main]
   workflow_dispatch:
 
 jobs:
-  build: # make sure build/ci work properly
+  build:
+    # make sure build/ci work properly
     runs-on: ubuntu-latest
     env:
       MAYHEM_TOKEN: ${{ secrets.MAYHEM_TOKEN }}
@@ -23,7 +24,8 @@ jobs:
       - run: |
           npm run all
 
-  test-some-outputs: # make sure the action works on a clean machine without building
+  test-some-outputs:
+    # make sure the action works on a clean machine without building
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -34,12 +36,13 @@ jobs:
       - uses: ./
         id: mcode-action
         with:
-          mayhem-url: https://beta.forallsecure.com
+          mayhem-url: ${{ secrets.MAYHEM_URL }}
           mayhem-token: ${{ secrets.MAYHEM_TOKEN }}
           github-token: ${{ secrets.GITHUB_TOKEN }}
           sarif-output: out/sarif/
-          args: --image forallsecure/lighttpd:vulnerable --file __tests__/Mayhemfile --duration 60
-          
+          package: __tests__/lighttpd
+          args: --image forallsecure/lighttpd:vulnerable --duration 60
+
       - name: Upload SARIF file(s)
         uses: github/codeql-action/upload-sarif@v2
         with:
@@ -50,7 +53,8 @@ jobs:
           [ -z "${{ steps.mcode-action.outputs.runId }}" ] && echo "runId was blank!" && exit 1;
           echo "The run id was: ${{ steps.mcode-action.outputs.runId }}"
 
-  test-all-outputs: # make sure the action works on a clean machine without building
+  test-all-outputs:
+    # make sure the action works on a clean machine without building
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -61,13 +65,14 @@ jobs:
       - uses: ./
         id: mcode-action
         with:
-          mayhem-url: https://beta.forallsecure.com
+          mayhem-url: ${{ secrets.MAYHEM_URL }}
           mayhem-token: ${{ secrets.MAYHEM_TOKEN }}
           github-token: ${{ secrets.GITHUB_TOKEN }}
           junit-output: out/junit/
           sarif-output: out/sarif/
           coverage-output: out/coverage/
-          args: --image forallsecure/lighttpd:vulnerable --file __tests__/Mayhemfile --duration 60
+          package: __tests__/lighttpd
+          args: --image forallsecure/lighttpd:vulnerable --duration 60
 
       - name: Archive Coverage report
         uses: actions/upload-artifact@v3
@@ -82,7 +87,7 @@ jobs:
           name: mcode-junit
           path: out/junit/
           if-no-files-found: error
-          
+
       - name: Upload SARIF file(s)
         uses: github/codeql-action/upload-sarif@v2
         with:
@@ -93,7 +98,8 @@ jobs:
           [ -z "${{ steps.mcode-action.outputs.runId }}" ] && echo "runId was blank!" && exit 1;
           echo "The run id was: ${{ steps.mcode-action.outputs.runId }}"
 
-  test-no-outputs: # make sure the action works on a clean machine without building
+  test-no-outputs:
+    # make sure the action works on a clean machine without building
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -104,10 +110,11 @@ jobs:
       - uses: ./
         id: mcode-action
         with:
-          mayhem-url: https://beta.forallsecure.com
+          mayhem-url: ${{ secrets.MAYHEM_URL }}
           mayhem-token: ${{ secrets.MAYHEM_TOKEN }}
           github-token: ${{ secrets.GITHUB_TOKEN }}
-          args: --image forallsecure/lighttpd:vulnerable --file __tests__/Mayhemfile --duration 60
+          package: __tests__/lighttpd
+          args: --image forallsecure/lighttpd:vulnerable --duration 60
 
       - name: Print runId (${{ steps.mcode-action.outputs.runId }}) and test it's non-empty
         run: |
@@ -126,13 +133,14 @@ jobs:
       - uses: ./
         id: mcode-action
         with:
-          mayhem-url: https://beta.forallsecure.com
+          mayhem-url: ${{ secrets.MAYHEM_URL }}
           mayhem-token: ${{ secrets.MAYHEM_TOKEN }}
           github-token: ${{ secrets.GITHUB_TOKEN }}
-          args: --image forallsecure/lighttpd:vulnerable --file __tests__/Mayhemfile --duration 60
+          package: __tests__/lighttpd
+          args: --image forallsecure/lighttpd:vulnerable --duration 60
           # override the default owner which is forallsecure
-          # the secrets.MAYHEM_TOKEN is this case is owned by vlussenburg, which is why we choose that owner here
-          owner: vlussenburg
+          # the secrets.MAYHEM_TOKEN is this case is owned by mcode-action, which is why we choose that owner here
+          owner: mcode-action
           # override default verbosity which is info (contrary to what the `action.yml` says: https://github.com/ForAllSecure/mcode-action/blob/806778bb4a79d793f678087d0f9f3ff18f9a2d93/src/main.ts#L36)
           verbosity: debug
           # already covered and skipped in this test: mayhem-token, mayhem-url, github-token, sarif|junit|coverage-output, args
@@ -141,3 +149,32 @@ jobs:
         run: |
           [ -z "${{ steps.mcode-action.outputs.runId }}" ] && echo "runId was blank!" && exit 1;
           echo "The run id was: ${{ steps.mcode-action.outputs.runId }}"
+
+  test-fail-on-defects:
+    # make sure the action works on a clean machine without building
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          # fetch entire history to compute diffs between jobs
+          fetch-depth: 0
+
+      - uses: ./
+        id: mcode-action
+        continue-on-error: true
+        with:
+          mayhem-url: ${{ secrets.MAYHEM_URL }}
+          mayhem-token: ${{ secrets.MAYHEM_TOKEN }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          package: __tests__/mayhemit
+          args: --image forallsecure/c-base-executable:latest --duration 60
+          fail-on-defects: true
+
+      # Previous step should fail
+      - name: Invert success and failure
+        run: if [[ ${{ steps.mcode-action.outcome }} == "failure" ]]; then exit 0; else exit 1; fi
+
+      - name: Print runId (${{ steps.mcode-action.outputs.runId }}) and test it's non-empty
+        run: |
+          [ -z "${{ steps.mcode-action.outputs.runId }}" ] && echo "runId was blank!" && exit 1;
+          echo "The run id was: ${{ steps.mcode-action.outputs.runId }}"

__tests__/Mayhemfile __tests__/lighttpd/Mayhemfile__tests__/Mayhemfile renamed to __tests__/lighttpd/Mayhemfile

@@ -0,0 +1,9 @@
+image: forallsecure/c-base-executable          # fields are unnecessary
+duration: 60                                   # since they will
+project: forallsecure/mayhemit                 # be filled at
+target: mayhemit                               # run creation time
+tasks:
+  # just do one quick task
+  - name: regression_testing
+cmds:
+  - cmd: /mayhemit @@

__tests__/mayhemit/Mayhemfile

@@ -0,0 +1 @@
+bug

__tests__/mayhemit/testsuite/crash

@@ -32,6 +32,10 @@ inputs:
   verbosity:
     description: verbosity level for starting runs
     required: false
+  package:
+    description: Path to the Mayhem package relative to the repository root
+    required: false
+    default: "."
   args:
     description: command line arguments to override CLI behavior
     required: false

action.yml

@@ -30,6 +30,7 @@ async function run(): Promise<void> {
       required: true,
     });
     const mayhemToken: string = core.getInput("mayhem-token") || githubToken;
+    const packagePath: string = core.getInput("package") || ".";
     const sarifOutput: string = core.getInput("sarif-output") || "";
     const junitOutput: string = core.getInput("junit-output") || "";
     const coverageOutput: string = core.getInput("coverage-output") || "";
@@ -117,7 +118,7 @@ async function run(): Promise<void> {
     fi
 
     # Run mayhem
-    run=$(${cli} --verbosity ${verbosity} run . \
+    run=$(${cli} --verbosity ${verbosity} run ${packagePath} \
                  --project ${repo.toLowerCase()} \
                  --owner ${owner} ${argsString});
 
@@ -132,7 +133,10 @@ async function run(): Promise<void> {
     fi
 
     # if the user didn't specify requiring any output, don't wait for the result.
-    if [ -z "${coverageOutput}" ] && [ -z "${junitOutput}" ] && [ -z "${sarifOutput}" ]; then
+    if [ -z "${coverageOutput}" ] && \
+        [ -z "${junitOutput}" ] && \
+        [ -z "${sarifOutput}" ] && \
+        [ "${failOnDefects.toString().toLowerCase()}" != "true" ]; then
       echo "No coverage, junit or sarif output requested, not waiting for job result.";
       exit 0;
     fi