Merge pull request #2752 from antmicro/umarcor/gcp
ci: move uploading of artifacts from workflow Tests to Automerge
diff --git a/.github/workflows/Automerge.yml b/.github/workflows/Automerge.yml
index 5a885d7..1628268 100644
--- a/.github/workflows/Automerge.yml
+++ b/.github/workflows/Automerge.yml
@@ -14,7 +14,105 @@
Pipeline:
if: ${{ !(github.event_name != 'pull_request' && github.actor == 'dependabot[bot]') }}
uses: ./.github/workflows/Tests.yml
- secrets: inherit
+
+
+ GCS:
+ if: ${{ !(github.event_name != 'pull_request' && github.actor == 'dependabot[bot]') }}
+ needs: Pipeline
+ container: ubuntu:bionic
+ runs-on:
+ - self-hosted
+ - Linux
+ - X64
+
+ env:
+ MAX_CORES: 80
+ GHA_EXTERNAL_DISK: "tools"
+ GHA_SA: "gh-sa-f4pga-arch-defs-ci"
+
+ steps:
+
+ - name: '📤 Download artifact: arch-defs packages'
+ uses: actions/download-artifact@v3
+ with:
+ name: packages
+ path: './'
+
+ - name: 🕐 Timestamp, Hash and packages.list
+ id: timestamp
+ run: |
+ TIMESTAMP="$(date +'%Y%m%d-%H%M%S')"
+ echo '::set-output name=timestamp::'"$TIMESTAMP"
+
+ GITHUB_STEP_SUMMARY=${GITHUB_STEP_SUMMARY:-github_step_summary.md}
+ echo 'Timestamp: $TIMESTAMP' >> $GITHUB_STEP_SUMMARY
+ echo 'Hash: '"$(echo symbiflow-arch-defs*.tar.xz | sed 's/.*-\(.*\)\.tar\.xz/\1/')" >> $GITHUB_STEP_SUMMARY
+
+ ls *.tar.xz > packages.list
+
+ mkdir -p install
+
+ - name: '🐍 Setup Python'
+ run: |
+ apt update -qq
+ apt install -y python3
+
+ - name: 🚀 Upload to Google Cloud Storage (GCS) bucket
+ run: |
+ python3 - <<EOF
+ from subprocess import check_call, run
+ from pathlib import Path
+ from re import compile as re_compile
+ TIMESTAMP = '${{ steps.timestamp.outputs.timestamp }}'
+ DO_UPLOAD = '${{ github.event_name == 'push' && github.ref_name == 'main' }}' == 'true'
+ print(f'> TIMESTAMP: {TIMESTAMP}')
+ PACKAGE_RE = re_compile("symbiflow-arch-defs-([a-zA-Z0-9_-]+)-([a-z0-9])")
+ with (Path(__file__).parent.parent.parent / 'packages.list').open('r') as rptr:
+ for artifact in rptr.read().splitlines():
+ if DO_UPLOAD:
+ check_call([
+ 'gsutil', 'cp', str(artifact),
+ f'gs://symbiflow-arch-defs/artifacts/prod/foss-fpga-tools/symbiflow-arch-defs/continuous/install/{TIMESTAMP}/'
+ ])
+ m = PACKAGE_RE.match(artifact)
+ assert m, f"Package name not recognized! {artifact}"
+ package_name = m.group(1)
+ if package_name == "install":
+ package_name = "toolchain"
+ with (Path("install") / f"symbiflow-{package_name}-latest").open("w") as wptr:
+ wptr.write(
+ 'https://storage.googleapis.com/symbiflow-arch-defs/artifacts/prod/foss-fpga-tools/'
+ f'symbiflow-arch-defs/continuous/install/{TIMESTAMP}/{artifact}'
+ )
+ if DO_UPLOAD:
+ run('gsutil -m rsync $* -r install gs://${{ secrets.GCP_STORAGE_BUCKET }}/', shell=True, check=False)
+ EOF
+
+ - name: '📤 Upload artifact: latest'
+ uses: actions/upload-artifact@v3
+ with:
+ name: latest
+ path: install
+
+
+ Assets:
+ if: ${{ !(github.event_name != 'pull_request' && github.actor == 'dependabot[bot]') && (github.event_name == 'push' && github.ref_name == 'main')}}
+ needs: GCS
+ runs-on: ubuntu-latest
+
+ steps:
+
+ - name: '📤 Download artifact: latest'
+ uses: actions/download-artifact@v3
+ with:
+ name: latest
+ path: install
+
+ - uses: pyTooling/Actions/releaser@r0
+ with:
+ token: ${{ github.token }}
+ tag: latest
+ files: install/*
Automerge:
diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml
index 9c1c0e3..31427fc 100644
--- a/.github/workflows/Tests.yml
+++ b/.github/workflows/Tests.yml
@@ -63,7 +63,6 @@
env:
MAX_CORES: 80
GHA_EXTERNAL_DISK: "tools"
- GHA_SA: "gh-sa-f4pga-arch-defs-ci"
steps:
@@ -75,53 +74,10 @@
- name: '🚧 Execute test script'
run: stdbuf -i0 -o0 -e0 ./.github/scripts/install.sh
- - name: 🕐 Timestamp, Hash and packages.list
- id: timestamp
- run: |
- TIMESTAMP="$(date +'%Y%m%d-%H%M%S')"
- echo '::set-output name=timestamp::'"$TIMESTAMP"
-
- GITHUB_STEP_SUMMARY=${GITHUB_STEP_SUMMARY:-github_step_summary.md}
- echo 'Timestamp: $TIMESTAMP' >> $GITHUB_STEP_SUMMARY
- echo 'Hash: '"$(echo symbiflow-arch-defs*.tar.xz | sed 's/.*-\(.*\)\.tar\.xz/\1/')" >> $GITHUB_STEP_SUMMARY
-
- ls *.tar.xz > packages.list
-
- mkdir -p install
-
- - name: 🚀 Upload to Google Cloud Storage (GCS) bucket
- if: github.event_name == 'push' && github.ref_name == 'main'
- run: |
- python3 - <<EOF
- from subprocess import check_call
- from pathlib import Path
- from re import compile as re_compile
- TIMESTAMP = '${{ steps.timestamp.outputs.timestamp }}'
- print(f'> TIMESTAMP: {TIMESTAMP}')
- PACKAGE_RE = re_compile("symbiflow-arch-defs-([a-zA-Z0-9_-]+)-([a-z0-9])")
- with (Path(__file__).parent.parent.parent / 'packages.list').open('r') as rptr:
- for artifact in rptr.read().splitlines():
- check_call([
- 'gsutil', 'cp', str(artifact),
- f'gs://symbiflow-arch-defs/artifacts/prod/foss-fpga-tools/symbiflow-arch-defs/continuous/install/{TIMESTAMP}/'
- ])
- m = PACKAGE_RE.match(artifact)
- assert m, f"Package name not recognized! {artifact}"
- package_name = m.group(1)
- if package_name == "install":
- package_name = "toolchain"
- with (Path("install") / f"symbiflow-{package_name}-latest").open("w") as wptr:
- wptr.write(
- 'https://storage.googleapis.com/symbiflow-arch-defs/artifacts/prod/foss-fpga-tools/'
- f'symbiflow-arch-defs/continuous/install/{TIMESTAMP}/{artifact}'
- )
- check_call('gsutil -m rsync $* -r install gs://${{ secrets.GCP_STORAGE_BUCKET }}/', shell=True)
- EOF
-
- name: '📤 Upload artifact: arch-defs packages'
uses: actions/upload-artifact@v3
with:
- name: ${{ steps.timestamp.outputs.timestamp }}
+ name: packages
path: '*.tar.xz'
- name: '📤 Upload artifact: results and plot'
