diff options
author | Kent Overstreet <kent.overstreet@linux.dev> | 2023-05-12 19:53:19 -0400 |
---|---|---|
committer | Kent Overstreet <kent.overstreet@linux.dev> | 2023-05-12 20:12:18 -0400 |
commit | 2beb85c7d95cee1d2121321c227890c7a0bc82ef (patch) | |
tree | 2a4f21da05563e7d5628f357f13490675c0bc261 | |
parent | c9217d9eb6e824e6e916194ab528ef441a6eb094 (diff) |
ci: Kill test-git-branch wrapper
Previously, we had a wrapper script around test-git-branch that always
fetched the latest version from the server and ran that.
But this appears to cause errors due to bash running out of file
descriptors, so this patch drops it.
Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>
-rw-r--r-- | ci/_test-git-branch.sh | 150 | ||||
-rwxr-xr-x | ci/test-git-branch.sh | 159 |
2 files changed, 158 insertions, 151 deletions
diff --git a/ci/_test-git-branch.sh b/ci/_test-git-branch.sh deleted file mode 100644 index 980fbbc..0000000 --- a/ci/_test-git-branch.sh +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env bash - -set -o nounset -set -o errexit -set -o errtrace - -ktest_verbose=1 - -KTEST_DIR=$(dirname "$(readlink -e "$0")")/.. -JOBSERVER_LINUX_REPO=ssh://$JOBSERVER/$JOBSERVER_HOME/linux - -. $KTEST_DIR/lib/common.sh - -ssh() { - ( - set +o errexit - - while true; do - env ssh "$@" - (($? == 0)) && break - sleep 1 - tput cuu1 - tput el - done - ) -} - -git_fetch() -{ - local repo=$1 - shift - - ( - set +o errexit - - while true; do - git fetch ssh://$JOBSERVER/$JOBSERVER_HOME/$repo $@ - (($? == 0)) && break - sleep 1 - done - ) -} - -sync_git_repos() -{ - local repo - - for repo in ${JOBSERVER_GIT_REPOS[@]}; do - (cd ~/$repo; git_fetch $repo && git checkout -f FETCH_HEAD) || true > /dev/null - done -} - -echo "Getting test job" - -while true; do - TEST_JOB=( $(ssh $JOBSERVER get-test-job) ) - - [[ ${#TEST_JOB[@]} != 0 ]] && break - - sleep 30 -done - -BRANCH=${TEST_JOB[0]} -COMMIT=${TEST_JOB[1]} -TEST_PATH=${TEST_JOB[2]} -TEST_NAME=$(basename -s .ktest $TEST_PATH) -SUBTESTS=( "${TEST_JOB[@]:3}" ) -OUTPUT=$JOBSERVER_OUTPUT_DIR/$COMMIT - -if [[ -z $BRANCH ]]; then - echo "Error getting test job: need git branch" - exit 1 -fi - -if [[ -z $COMMIT ]]; then - echo "Error getting test job: need git commit" - exit 1 -fi - -if [[ -z $TEST_PATH ]]; then - echo "Error getting test job: need test to run" - exit 1 -fi - -echo "Running test $TEST_NAME for branch $BRANCH and commit $COMMIT" - -run_quiet "Syncing git repos" sync_git_repos - -run_quiet "Fetching $COMMIT" git_fetch linux $COMMIT -run_quiet "Checking out $COMMIT" git checkout FETCH_HEAD - -rm -rf ktest-out/out -mkdir -p ktest-out/out - -# Mark tests as not run: -for t in ${SUBTESTS[@]}; do - t=$(echo "$t"|tr / .) - - mkdir ktest-out/out/$TEST_NAME.$t - echo "IN PROGRESS" > ktest-out/out/$TEST_NAME.$t/status -done - -make -C "$KTEST_DIR/lib" supervisor - -while (( ${#SUBTESTS[@]} )); do - FULL_LOG=$TEST_NAME.$(hostname).$(date -Iseconds).log - - for t in ${SUBTESTS[@]}; do - FNAME=$(echo "$t"|tr / .) - ln -sfr "ktest-out/out/$FULL_LOG.br" \ - "ktest-out/out/$TEST_NAME.$FNAME/full_log.br" - done - - echo "Running test $TEST_NAME ${SUBTESTS[@]}" - - $KTEST_DIR/lib/supervisor -T 1200 -f "$FULL_LOG" -S -F \ - -b $TEST_NAME -o ktest-out/out \ - -- build-test-kernel run -P $TEST_PATH ${SUBTESTS[@]} & - wait - - SUBTESTS_REMAINING=() - - t=${SUBTESTS[0]} - FNAME=$(echo "$t"|tr / .) - - if grep -q "IN PROGRESS" ktest-out/out/$TEST_NAME.$FNAME/status; then - echo "NOT STARTED" > ktest-out/out/$TEST_NAME.$FNAME/status - fi - - for t in ${SUBTESTS[@]:1}; do - FNAME=$(echo "$t"|tr / .) - - if grep -q "IN PROGRESS" ktest-out/out/$TEST_NAME.$FNAME/status; then - SUBTESTS_REMAINING+=($t) - fi - done - - echo "Compressing output" - find ktest-out/out -type f -name \*log -print0|xargs -0 brotli --rm -9 - - ssh $JOBSERVER mkdir -p $OUTPUT - - echo "Sending results to jobserver" - (cd ktest-out/out; tar --create --file - *)| - ssh $JOBSERVER "(cd $OUTPUT; tar --extract --file -)" - - ssh $JOBSERVER gen-commit-summary $COMMIT - - SUBTESTS=( "${SUBTESTS_REMAINING[@]}" ) -done diff --git a/ci/test-git-branch.sh b/ci/test-git-branch.sh index a3d022e..81e3dde 100755 --- a/ci/test-git-branch.sh +++ b/ci/test-git-branch.sh @@ -1,7 +1,164 @@ #!/usr/bin/env bash +set -o nounset +set -o errexit +set -o errtrace + JOBSERVER=$1 +ktest_verbose=1 + +KTEST_DIR=$(dirname "$(readlink -e "$0")")/.. + +. $KTEST_DIR/lib/common.sh + +source <(ssh $JOBSERVER cat .ktestrc) + +JOBSERVER_LINUX_REPO=ssh://$JOBSERVER/$JOBSERVER_HOME/linux + +ssh() { + ( + set +o errexit + + while true; do + env ssh "$@" + (($? == 0)) && break + sleep 1 + tput cuu1 + tput el + done + ) +} + +git_fetch() +{ + local repo=$1 + shift + + ( + set +o errexit + + while true; do + git fetch ssh://$JOBSERVER/$JOBSERVER_HOME/$repo $@ + (($? == 0)) && break + sleep 1 + done + ) +} + +sync_git_repos() +{ + local repo + + for repo in ${JOBSERVER_GIT_REPOS[@]}; do + (cd ~/$repo; git_fetch $repo && git checkout -f FETCH_HEAD) || true > /dev/null + done +} + +run_test_job() +{ + BRANCH="$1" + COMMIT="$2" + TEST_PATH="$3" + shift 3 + SUBTESTS=("$@") + + TEST_NAME=$(basename -s .ktest $TEST_PATH) + OUTPUT=$JOBSERVER_OUTPUT_DIR/$COMMIT + + if [[ -z $BRANCH ]]; then + echo "Error getting test job: need git branch" + exit 1 + fi + + if [[ -z $COMMIT ]]; then + echo "Error getting test job: need git commit" + exit 1 + fi + + if [[ -z $TEST_PATH ]]; then + echo "Error getting test job: need test to run" + exit 1 + fi + + echo "Running test $TEST_NAME for branch $BRANCH and commit $COMMIT" + + run_quiet "Syncing git repos" sync_git_repos + + run_quiet "Fetching $COMMIT" git_fetch linux $COMMIT + run_quiet "Checking out $COMMIT" git checkout FETCH_HEAD + + rm -rf ktest-out/out + mkdir -p ktest-out/out + + # Mark tests as not run: + for t in ${SUBTESTS[@]}; do + t=$(echo "$t"|tr / .) + + mkdir ktest-out/out/$TEST_NAME.$t + echo "IN PROGRESS" > ktest-out/out/$TEST_NAME.$t/status + done + + make -C "$KTEST_DIR/lib" supervisor + + while (( ${#SUBTESTS[@]} )); do + FULL_LOG=$TEST_NAME.$(hostname).$(date -Iseconds).log + + for t in ${SUBTESTS[@]}; do + FNAME=$(echo "$t"|tr / .) + ln -sfr "ktest-out/out/$FULL_LOG.br" \ + "ktest-out/out/$TEST_NAME.$FNAME/full_log.br" + done + + echo "Running test $TEST_NAME ${SUBTESTS[@]}" + + $KTEST_DIR/lib/supervisor -T 1200 -f "$FULL_LOG" -S -F \ + -b $TEST_NAME -o ktest-out/out \ + -- build-test-kernel run -P $TEST_PATH ${SUBTESTS[@]} & + wait + + SUBTESTS_REMAINING=() + + t=${SUBTESTS[0]} + FNAME=$(echo "$t"|tr / .) + + if grep -q "IN PROGRESS" ktest-out/out/$TEST_NAME.$FNAME/status; then + echo "NOT STARTED" > ktest-out/out/$TEST_NAME.$FNAME/status + fi + + for t in ${SUBTESTS[@]:1}; do + FNAME=$(echo "$t"|tr / .) + + if grep -q "IN PROGRESS" ktest-out/out/$TEST_NAME.$FNAME/status; then + SUBTESTS_REMAINING+=($t) + fi + done + + echo "Compressing output" + find ktest-out/out -type f -name \*log -print0|xargs -0 brotli --rm -9 + + ssh $JOBSERVER mkdir -p $OUTPUT + + echo "Sending results to jobserver" + (cd ktest-out/out; tar --create --file - *)| + ssh $JOBSERVER "(cd $OUTPUT; tar --extract --file -)" + + ssh $JOBSERVER gen-commit-summary $COMMIT + + SUBTESTS=( "${SUBTESTS_REMAINING[@]}" ) + done +} + while true; do - source <(ssh $JOBSERVER cat .ktestrc bin/_test-git-branch.sh) + echo "Getting test job" + + while true; do + TEST_JOB=( $(ssh $JOBSERVER get-test-job) ) + + [[ ${#TEST_JOB[@]} != 0 ]] && break + + sleep 10 + done + + (run_test_job "${TEST_JOB[@]}") || sleep 10 done |