From f5a5044e207ba0b2dbb76308540405be0f22d5b8 Mon Sep 17 00:00:00 2001 From: Ivan Baravy Date: Thu, 16 Feb 2023 19:32:37 +0000 Subject: [PATCH] Implement new test runner, mainly for timeouts --- .gitignore | 1 + makefile | 20 ++-- test/runtests.c | 262 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 276 insertions(+), 7 deletions(-) create mode 100644 test/runtests.c diff --git a/.gitignore b/.gitignore index 34b4b2c..e2322b4 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,4 @@ apps/readdir apps/loader apps/asciivju apps/justawindow +test/runtests diff --git a/makefile b/makefile index 804d947..5c80ec8 100644 --- a/makefile +++ b/makefile @@ -50,10 +50,9 @@ endif ifeq ($(HOST),linux) all: umka_shell umka_fuse umka_os umka_gen_devices_dat umka.sym umka.prp \ - umka.lst tags default.skn skin.skn + umka.lst tags default.skn skin.skn test/runtests else ifeq ($(HOST),windows) -all: umka_shell umka.sym umka.prp \ - umka.lst default.skn skin.skn +all: umka_shell umka.sym umka.prp umka.lst default.skn skin.skn test/runtests else $(error your OS is not supported) endif @@ -102,9 +101,8 @@ $(HOST)/pci.o: $(HOST)/pci.c $(CC) $(CFLAGS_32) -std=gnu11 -c $< -o $@ deps/lodepng/lodepng.o: deps/lodepng/lodepng.c deps/lodepng/lodepng.h - $(CC) $(CFLAGS_32) -c $< -DLODEPNG_NO_COMPILE_ZLIB \ - -DLODEPNG_NO_COMPILE_DECODER -DLODEPNG_NO_COMPILE_DISK \ - -DLODEPNG_NO_COMPILE_ANCILLARY_CHUNKS -DLODEPNG_NO_COMPILE_CRC + $(CC) $(CFLAGS_32) -c $< -o $@ -DLODEPNG_NO_COMPILE_DECODER \ + -DLODEPNG_NO_COMPILE_ZLIB -DLODEPNG_NO_COMPILE_ANCILLARY_CHUNKS deps/isocline/src/isocline.o: deps/isocline/src/isocline.c \ deps/isocline/include/isocline.h @@ -181,10 +179,18 @@ umka_os.o: umka_os.c umka.h umka_gen_devices_dat.o: umka_gen_devices_dat.c umka.h $(CC) $(CFLAGS_32) -c $< +test/runtests: test/runtests.o deps/optparse/optparse.o + $(CC) $(LDFLAGS_32) -o $@ $^ + +test/runtests.o: test/runtests.c + $(CC) $(CFLAGS_32) -c $< -o $@ -Wno-deprecated-declarations + + .PHONY: all clean clean: rm -f umka_shell umka_fuse umka_os umka_gen_devices_dat umka.fas \ - umka.sym umka.lst umka.prp umka.cov coverage *.skn colors.dtp + umka.sym umka.lst umka.prp umka.cov coverage *.skn colors.dtp \ + test/runtests find . -name "*.o" -delete find . -name "*.a" -delete diff --git a/test/runtests.c b/test/runtests.c new file mode 100644 index 0000000..2b31218 --- /dev/null +++ b/test/runtests.c @@ -0,0 +1,262 @@ +/* + SPDX-License-Identifier: GPL-2.0-or-later + + UMKa - User-Mode KolibriOS developer tools + runtests - the test runner + + Copyright (C) 2023 Ivan Baravy +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "optparse/optparse.h" + +#ifndef PATH_MAX +#define PATH_MAX 0x1000 +#endif + +#define CMPFILE_BUF_LEN 0x100000 +#define TIMEOUT_FILENAME "timeout.txt" +#define TIMEOUT_STR_MAX_LEN 16 +#define ACTION_RUN 0 + +thread_local char bufa[CMPFILE_BUF_LEN]; +thread_local char bufb[CMPFILE_BUF_LEN]; +thread_local char outfname[PATH_MAX]; +thread_local char timeoutfname[PATH_MAX]; + +int silent_success = 1; + +static int +is_valid_test_name(const char *name) { + if (name[0] != 't') { + return 0; + } + for (size_t i = 1; i < 4 && name[i]; i++) { + if (name[1] < '0' || name[1] > '9') { + return 0; + } + } + return 1; +} + +static int +cmpfile(const char *fnamea, const char *fnameb) { + FILE *filea = fopen(fnamea, "rb"); + if (!filea) { + fprintf(stderr, "Can't open file '%s' for reading: %s\n", fnamea, + strerror(errno)); + return -1; + } + FILE *fileb = fopen(fnameb, "rb"); + if (!fileb) { + fprintf(stderr, "Can't open file '%s' for reading: %s\n", fnameb, + strerror(errno)); + return -1; + } + while (1) { + size_t reada = fread(bufa, CMPFILE_BUF_LEN, 1, filea); + size_t readb = fread(bufa, CMPFILE_BUF_LEN, 1, fileb); + if (reada != readb || memcmp(bufa, bufb, reada)) { + fprintf(stderr, "[!] files %s and %s don't match\n", fnamea, + fnameb); + return -1; + } + } + fclose(filea); + fclose(fileb); + return 0; +} + +static int +check_test_artefacts(const char *testname) { + DIR *tdir = opendir("."); + if (!tdir) { + fprintf(stderr, "Can't open directory '%s': %s\n", testname, + strerror(errno)); + return -1; + } + + struct dirent dent; + struct dirent *result; + const char *reffname; + while (readdir_r(tdir, &dent, &result)) { + reffname = dent.d_name; + const char *dotrefdot = strstr(reffname, ".ref."); + if (!dotrefdot) { + continue; + } + strcpy(outfname, reffname); + memcpy(outfname + (dotrefdot - reffname), ".out.", 5); + if (cmpfile(reffname, outfname)) { + return -1; + } + } + closedir(tdir); + return 0; +} + +struct test_wait_arg { + int pid; + pthread_cond_t *cond; +}; + +static unsigned +get_test_timeout(const char *testname) { + sprintf(timeoutfname, "%s/%s", testname, TIMEOUT_FILENAME); + FILE *f = fopen(timeoutfname, "rb"); + if (!f) { + fprintf(stderr, "[!] Can't open %s\n: %s\n", TIMEOUT_FILENAME, + strerror(errno)); + return 0; + } + char str[TIMEOUT_STR_MAX_LEN]; + fread(str, 1, TIMEOUT_STR_MAX_LEN, f); + fclose(f); + unsigned n, sec = 0, min = 0; + char *end; + n = strtoul(str, &end, 0); + switch (*end) { + case 's': + sec = n; + break; + case 'm': + min = n; + sec = strtoul(end+1, &end, 0); + break; + default: + fprintf(stderr, "[!] bad timeout value: %s\n", str); + return 0; + } + return min*60 + sec; +} + +#ifndef _WIN32 +static void * +thread_wait(void *arg) { + struct test_wait_arg *wa = arg; + int status; + waitpid(wa->pid, &status, 0); + pthread_cond_signal(wa->cond); + return (void *)(intptr_t)status; +} + +static void * +run_test(const void *arg) { + const char *test_name = arg; + void *result = NULL; + pthread_cond_t cond = PTHREAD_COND_INITIALIZER; + pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; + pthread_mutex_lock(&mutex); + int child; + if (!(child = fork())) { + chdir(test_name); + execl("../../umka_shell", "../../umka_shell", "-ri", "run.us", "-o", + "out.log", NULL); + fprintf(stderr, "Can't run test command: %s\n", strerror(errno)); + return (void *)-1; + } + pthread_t t; + struct test_wait_arg wa = {.pid = child, .cond = &cond}; + pthread_create(&t, NULL, thread_wait, &wa); + unsigned tout = get_test_timeout(test_name); + struct timespec ts; + clock_gettime(CLOCK_REALTIME, &ts); + ts.tv_sec += tout; + int status; + if ((status = pthread_cond_timedwait(&cond, &mutex, &ts))) { + fprintf(stderr, "[!] %s: timeout (%um%us)\n", test_name, + tout/60, tout % 60); + kill(child, SIGKILL); + result = (void *)(intptr_t)status; + } + pthread_join(t, &result); + pthread_mutex_unlock(&mutex); + + return result; +} +#else +static void * +run_test(void *arg) { + const char *test_name = arg; + + return (void *)-1; +} +#endif + +pthread_mutex_t readdir_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void * +thread_run_test(void *arg) { + DIR *cwd = arg; + struct dirent *dent; + while (1) { + pthread_mutex_lock(&readdir_mutex); + dent = readdir(cwd); + pthread_mutex_unlock(&readdir_mutex); + if (!dent) { + break; + } + const char *testname = dent->d_name; + if (!is_valid_test_name(testname)) { + continue; + } + fprintf(stderr, "running test %s\n", testname); + if (run_test(testname) || check_test_artefacts(testname)) { + fprintf(stderr, "[!] test %s failed\n", testname); + } else if (!silent_success) { + fprintf(stderr, "test %s ok\n", testname); + } + } + return NULL; +} + +int +main(int argc, char *argv[]) { + (void)argc; +// int action = ACTION_RUN; + size_t nthreads = 1; + + struct optparse opts; + optparse_init(&opts, argv); + int opt; + + while ((opt = optparse(&opts, "j:s")) != -1) { + switch (opt) { + case 'j': + nthreads = strtoul(opts.optarg, NULL, 0); + break; + case 's': + silent_success = 0; + break; + default: + fprintf(stderr, "[!] Unknown option: '%c'\n", opt); + exit(1); + } + } + + + DIR *cwd = opendir("."); + pthread_t *threads = malloc(sizeof(pthread_t)*nthreads); + for (size_t i = 0; i < nthreads; i++) { + pthread_create(threads + i, NULL, thread_run_test, cwd); + } + for (size_t i = 0; i < nthreads; i++) { + void *retval; + pthread_join(threads[i], &retval); + } + free(threads); + return 0; +}