-
iotop is written in Python, which I cannot support on my embedded router.
looking for an alternative, written in C.
is there anything? :-//
before I start writing by myself
-
https://github.com/Tomas-M/iotop
Inspired by iotop Python script from Guillaume Chazarain, rewritten in C by Vyacheslav Trushkin and improved by Boian Bonev so it runs without Python at all.
-
https://github.com/Tomas-M/iotop
Inspired by iotop Python script from Guillaume Chazarain, rewritten in C by Vyacheslav Trushkin and improved by Boian Bonev so it runs without Python at all.
thanks!
have you already tried it?
Going to add it to the new Catalyst-stage4 profile ;D
it will be tested next week!
-
Yes, just compiled it and tried on Arch Linux.
It seems to work fine so far.
-
so, iotop-c compiled on different machines, and
1) it perfectly works on my PC, i686-GNU-Linux
2) it doesn't correctly work on my router, MIPS32/LE-GNU-Linux
reports are always zero, and something triggers prIO to fail with an error
struct xxxid_stats *ms=cs->sor[i],*s;
...
if (!config.f.hideprio)
{
char c=' ';
if (k==-1&&th_prio_diff)
{
c='!';
}
if (s->error_i) <---------------- something triggers this
{
attron(config.f.nocolor?A_ITALIC:COLOR_PAIR(RED_PAIR));
printw("tp_#1 ");
attroff(config.f.nocolor?A_ITALIC:COLOR_PAIR(RED_PAIR));
}
else
{
printw("%c%4s ",c,str_ioprio(s->io_prio));
}
}
(src/view_curses.c)
-
I don't know how active the author is, but you should open a ticket on github with this.
-
Could you try running the following program, as shown on the third line (if you save it as this.c)?
// Spdx-License-Identifier: CC0-1.0
//
// gcc -Wall -O2 this.c -o showioprio && ./showioprio $(cd /proc && /bin/ls -1d [1-9]* | tr -d /) | sort -n
#define _GNU_SOURCE
#define _POSIX_C_SOURCE 200809L
#include <stdlib.h>
#include <unistd.h>
#include <sys/syscall.h>
#include <sys/resource.h>
#include <sched.h>
#include <string.h>
#include <errno.h>
#include <stdio.h>
int report(pid_t p)
{
errno = 0;
long val = syscall(SYS_ioprio_get, /*IOPRIO_WHO_PROCESS*/ 1, (int)p);
if (errno) {
if (errno == ESRCH) {
fprintf(stderr, "%d: No such process.\n", (int)p);
return 0;
} else {
fprintf(stderr, "%d: ioprio_get syscall failed: %s.\n", (int)p, strerror(errno));
return -1;
}
}
long c = val >> 13;
if (c == 0) {
errno = 0;
int n = getpriority(PRIO_PROCESS, p);
if (errno) {
fprintf(stderr, "%d: getpriority() failed: %s.\n", (int)p, strerror(errno));
return -1;
}
if (n < -20 || n > 19) {
fprintf(stderr, "%d: getpriority() returned an invalid scheduling priority, %d.\n", (int)p, n);
return -1;
}
int v = (n + 20) / 5; /* Specified as the mapping between scheduling priority and IO priority */
int s = sched_getscheduler(p);
switch (s & 0x3FFFFFFF) {
case -1:
fprintf(stderr, "%d: sched_getscheduler() failed: %s.\n", (int)p, strerror(errno));
return -1;
case SCHED_FIFO:
case SCHED_RR:
case SCHED_DEADLINE:
printf("%d: Realtime %d (default)\n", (int)p, v);
break;
case SCHED_IDLE:
printf("%d: Idle %d (default)\n", (int)p, v);
break;
default:
printf("%d: Best-effort %d (default)\n", (int)p, v);
break;
}
fflush(stdout);
return 0;
} else {
int v = val & 0x1FFF;
if (v < 0 || v > 7) {
printf("%d: Unknown ioprio 0x%lx\n", (int)p, (unsigned long)val);
fflush(stdout);
return -1;
}
switch (c) {
case 1: printf("%d: Realtime %d\n", (int)p, v);
break;
case 2: printf("%d: Best-effort %d\n", (int)p, v);
break;
case 3: printf("%d: Idle %d\n", (int)p, v);
break;
default: fprintf(stderr, "%d: Unknown ioprio 0x%lx\n", (int)p, (unsigned long)val);
return -1;
}
fflush(stdout);
return 0;
}
}
int parse_pid(const char *src, pid_t *to)
{
const char *end;
long val;
if (!src || !*src)
return errno = EINVAL;
errno = 0;
end = src;
val = strtol(src, (char **)(&end), 0);
if (errno)
return errno;
if (src == end || *end || (long)((pid_t)val) != val)
return errno = EINVAL;
if (to)
*to = (pid_t)val;
return 0;
}
int main(int argc, char *argv[])
{
pid_t p;
if (argc == 2 && (!strcmp(argv[1], "-h") || !strcmp(argv[1], "--help"))) {
fprintf(stderr, "\n");
fprintf(stderr, "Usage: %s -h | --help\n", argv[0]);
fprintf(stderr, " %s PID [ PID ... ]\n", argv[0]);
fprintf(stderr, "\n");
fprintf(stderr, "This reports the IO class and priority for the specified process or processes.\n");
fprintf(stderr, "\n");
}
if (argc < 2) {
if (report(getpid()))
return EXIT_FAILURE;
return EXIT_SUCCESS;
}
for (int arg = 1; arg < argc; arg++) {
if (parse_pid(argv[arg], &p) || p < 1) {
fprintf(stderr, "%s: Invalid process ID: %s.\n", argv[arg], strerror(errno));
return EXIT_FAILURE;
}
if (report(p))
return EXIT_FAILURE;
fflush(stdout);
}
return EXIT_SUCCESS;
}
It uses the same logic as the current kernel (as of 2023-03-17): class zero indicates I/O priority is derived from CPU priority via (nice+20)/5 (in which case the value part is irrelevant); if CPU scheduling policy is SCHED_IDLE, then the I/O priority class is idle also; if SCHED_FIFO, SCHED_RR, or SCHED_DEADLINE, then the I/O priority class is realtime; otherwise the I/O priority class is best effort.
All processes running on my desktop machine report one of
Best-effort 0 (default)
Best-effort 1 (default)
Best-effort 2 (default)
Best-effort 3
Best-effort 3 (default)
Best-effort 4 (default)
Best-effort 5 (default)
Best-effort 7 (default)
Realtime 4 (default)
where the (default) just denotes that the I/O priority was not set explicitly, but was inherited from the default CPU priority as mentioned above.
Running the example command it is normal for it to report three or so PIDs as not existing, because they have been reaped before the example program runs.
Any error messages output to standard error would be interesting; I only get three "No such process." ones.
In particular, check out for "Function not implemented" and "Operation not supported" errors.
-
Just remembered that it requires elevated rights to work properly. So check that it runs with proper rights on your router, which I suspect it doesn't.
Quoting from the output of iotop (it doesn't seem to be documented in the main README of the project, which I think has been overlooked.)
The Linux kernel interfaces that iotop relies on now require root privileges
or the NET_ADMIN capability. This change occurred because a security issue
(CVE-2011-2494) was found that allows leakage of sensitive data across user
boundaries. If you require the ability to run iotop as a non-root user, please
configure sudo to allow you to run iotop as root.
Alternatively to using sudo the NET_ADMIN capability can be set by
$ sudo setcap 'cap_net_admin+eip' <path-to>/iotop
Be warned that this will also allow other users to run it and get access to
information that normally should not be available to them.
Please do not file bugs on iotop about this.
-
Note that iotop only sets the error_i member in one place, in src/xxxid_info.c:make_stats() (https://github.com/Tomas-M/iotop/blob/master/src/xxxid_info.c#L242), and only when src/ioprio.c:get_ioprio() (https://github.com/Tomas-M/iotop/blob/master/src/ioprio.c) returns -1. And it can only return -1 if syscall(SYS_ioprio_get, ...) returns an error (-1, in which case errno is set to the error, but not reported by iotop).
If it is a permission issue, my test program will complain "PID: ioprio_get syscall failed: Permission denied".
FWIW, I'm running a 5.4.0 kernel, and it does not require privileges for the priority information, although security modules can change that. Other stuff in iotop, specifically taskstats netlink does, require NET_ADMIN.
-
Could you try running the following program, as shown on the third line (if you save it as this.c)?
renamed test.c as showioprio.c, added Makefile
# make
showioprio.c: In function 'report':
showioprio.c:51: error: 'SCHED_DEADLINE' undeclared (first use in this function)
showioprio.c:51: error: (Each undeclared identifier is reported only once
showioprio.c:51: error: for each function it appears in.)
showioprio.c:55: error: 'SCHED_IDLE' undeclared (first use in this function)
showioprio.c: In function 'main':
showioprio.c:132: error: 'for' loop initial declaration used outside C99 mode
make: *** [all] Error 1
{ SCHED_DEADLINE, SCHED_IDLE } undeclared :-//
running kernel v6.2.0, I have already updated all the linux-headers files
[ U ] sys-kernel/linux-headers-5.4-r99 [2.6.27-r2]
but it didn't help
-
Just remembered that it requires elevated rights to work properly. So check that it runs with proper rights on your router, which I suspect it doesn't.
runs as UID=0 (root)
-
#define _GNU_SOURCE
#define _POSIX_C_SOURCE 200809L
#include <stdlib.h>
#include <unistd.h>
#include <sys/syscall.h>
#include <sys/resource.h>
#include <sched.h>
#include <string.h>
#include <errno.h>
#include <stdio.h>
int report
(
pid_t p
)
{
errno = 0;
long val = syscall(SYS_ioprio_get, /*IOPRIO_WHO_PROCESS*/ 1, (int) p);
if (errno)
{
if (errno == ESRCH)
{
fprintf(stderr, "%d: No such process.\n", (int) p);
return 0;
}
else
{
fprintf(stderr, "%d: ioprio_get syscall failed: %s.\n", (int) p, strerror(errno));
return -1;
}
}
long c = val >> 13;
if (c == 0)
{
errno = 0;
int n = getpriority(PRIO_PROCESS, p);
if (errno)
{
fprintf(stderr, "%d: getpriority() failed: %s.\n", (int) p, strerror(errno));
return -1;
}
if (n < -20 || n > 19)
{
fprintf(stderr, "%d: getpriority() returned an invalid scheduling priority, %d.\n", (int) p, n);
return -1;
}
int v = (n + 20) / 5; /* Specified as the mapping between scheduling priority and IO priority */
int s = sched_getscheduler(p);
switch (s & 0x3FFFFFFF)
{
case -1:
fprintf(stderr, "%d: sched_getscheduler() failed: %s.\n", (int) p, strerror(errno));
return -1;
// case SCHED_FIFO:
// case SCHED_RR:
// case SCHED_DEADLINE:
// printf("%d: Realtime %d (default)\n", (int)p, v);
// break;
//
// case SCHED_IDLE:
// printf("%d: Idle %d (default)\n", (int)p, v);
// break;
default:
printf("%d: Best-effort %d (default)\n", (int) p, v);
break;
}
fflush(stdout);
return 0;
}
else
{
int v = val & 0x1FFF;
if (v < 0 || v > 7)
{
printf("%d: Unknown ioprio 0x%lx\n", (int) p, (unsigned long) val);
fflush(stdout);
return -1;
}
switch (c)
{
case 1: printf("%d: Realtime %d\n", (int) p, v);
break;
case 2: printf("%d: Best-effort %d\n", (int) p, v);
break;
case 3: printf("%d: Idle %d\n", (int) p, v);
break;
default: fprintf(stderr, "%d: Unknown ioprio 0x%lx\n", (int) p, (unsigned long) val);
return -1;
}
fflush(stdout);
return 0;
}
}
int parse_pid
(
char *src,
pid_t *to
)
{
char *end;
long val;
if (!src || !*src)
{
return errno = EINVAL;
}
errno = 0;
end = src;
val = strtol(src, (char * *) (&end), 0);
if (errno)
{
return errno;
}
if (src == end || *end || (long) ((pid_t) val) != val)
{
return errno = EINVAL;
}
if (to)
{
*to = (pid_t) val;
}
return 0;
}
int main
(
int argc,
char *argv[]
)
{
pid_t p;
int arg;
if (argc == 2 && (!strcmp(argv[1], "-h") || !strcmp(argv[1], "--help")))
{
fprintf(stderr, "\n");
fprintf(stderr, "Usage: %s -h | --help\n", argv[0]);
fprintf(stderr, " %s PID [ PID ... ]\n", argv[0]);
fprintf(stderr, "\n");
fprintf(stderr, "This reports the IO class and priority for the specified process or processes.\n");
fprintf(stderr, "\n");
}
if (argc < 2)
{
if (report(getpid()))
{
return EXIT_FAILURE;
}
return EXIT_SUCCESS;
}
for (arg = 1; arg < argc; arg++)
{
if (parse_pid(argv[arg], &p) || p < 1)
{
fprintf(stderr, "%s: Invalid process ID: %s.\n", argv[arg], strerror(errno));
return EXIT_FAILURE;
}
if (report(p))
{
return EXIT_FAILURE;
}
fflush(stdout);
}
return EXIT_SUCCESS;
}
commenting some lines, it compiles and reports something interesting
16533: ioprio_get syscall failed: Function not implemented.
-
is it glibc?
tried on
- MacMini-G4 with sys-libs/glibc-v2.11.2 (~2009): ioprio_get syscall success
- Mips32r2 router with sys-libs/glibc-v2.9 (~2008): ioprio_get syscall failed: Function not implemented
-
- #include <sched.h>
+ #include <linux/sched.h>
still it fails, but this way it compiles
- - -
...
/*
* Scheduling policies
*/
#define SCHED_NORMAL 0
#define SCHED_FIFO 1
#define SCHED_RR 2
#define SCHED_BATCH 3
/* SCHED_ISO: reserved but not implemented yet */
#define SCHED_IDLE 5
#define SCHED_DEADLINE 6
...
/usr/include/linux/sched.h -> sys-kernel/linux-headers-5.4-r99, defines SCHED_*
/usr/include/sched.h -> sys-libs/glibc-2.9, doesn't define SCHED_*
so *is* "linux/sched.h" correct? :-//
-
16533: ioprio_get syscall failed: Function not implemented.
Your kernel does not have any I/O schedulers. This is perfectly normal if you have CONFIG_BLOCK=n, i.e. a kernel without support for block devices.
The ioprio_get() syscall is implemented in block/ioprio.c:SYSCALL_DEFINE2(ioprio_get, ...) (https://elixir.bootlin.com/linux/latest/source/block/ioprio.c), which AFAIK is implemented iff CONFIG_BLOCK=y or =m.
-
...
CONFIG_RT_MUTEXES=y
CONFIG_BASE_SMALL=0
# CONFIG_MODULES is not set
CONFIG_BLOCK=y
CONFIG_BLOCK_LEGACY_AUTOLOAD=y
CONFIG_BLK_ICQ=y
...
weird, because this is the config :o :o :o
-
# cat /sys/block/sda/queue/scheduler
none [mq-deadline] kyber bfq
:-//
-
so, if it's not glibc ...
cat /proc/kallsyms | grep ioprio_get
80354c3c T sys_ioprio_get
80354c3c T __se_sys_ioprio_get
(added debug-symbols, recompiled)
I am perplexed ... ioprio_get is there, but the userspace doesn't see it :-//
-
#include <stdio.h>
#include <sys/syscall.h>
#include <errno.h>
#include <string.h>
void test
(
long arg
)
{
int ret;
ret = syscall(arg);
printf("syscall(%ld) = %d (%s)\n", arg, ret, strerror(errno));
}
int main()
{
long i0;
for (i0 = 1; i0 < 1000; i0++)
{
test(i0);
}
return 0;
}
wrote this program: all the system calls "syscall" failed!
it seems syscall() is not working at all :-//
-
syscall fails with what error? Always the same?
-
syscall fails with what error? Always the same?
syscall(1) = -1 (Function not implemented)
syscall(2) = -1 (Function not implemented)
syscall(3) = -1 (Function not implemented)
syscall(4) = -1 (Function not implemented)
syscall(5) = -1 (Function not implemented)
syscall(6) = -1 (Function not implemented)
syscall(7) = -1 (Function not implemented)
syscall(8) = -1 (Function not implemented)
syscall(9) = -1 (Function not implemented)
syscall(10) = -1 (Function not implemented)
...
syscall(999) = -1 (Function not implemented)
-
Can you have a look at the /sys/kernel/debug/tracing/events/syscalls directory?
sudo ls /sys/kernel/debug/tracing/events/syscalls
(sudo is not necessary if it's running as root.)
-
/sys/kernel/debug/ is empty
-
Unless you can find an obvious cause for your issue, having access to kernel debug would be a nice help.
If this directory is empty, that probably means the kernel was not built with the CONFIG_DYNAMIC_DEBUG config flag. Can you rebuild it with this flag enabled?
-
Unless you can find an obvious cause for your issue, having access to kernel debug would be a nice help.
If this directory is empty, that probably means the kernel was not built with the CONFIG_DYNAMIC_DEBUG config flag. Can you rebuild it with this flag enabled?
already built with that :-//
#
# printk and dmesg options
#
# CONFIG_PRINTK_TIME is not set
# CONFIG_PRINTK_CALLER is not set
# CONFIG_STACKTRACE_BUILD_ID is not set
CONFIG_CONSOLE_LOGLEVEL_DEFAULT=7
CONFIG_CONSOLE_LOGLEVEL_QUIET=15
CONFIG_MESSAGE_LOGLEVEL_DEFAULT=4
# CONFIG_BOOT_PRINTK_DELAY is not set
CONFIG_DYNAMIC_DEBUG=y
CONFIG_DYNAMIC_DEBUG_CORE=y
CONFIG_SYMBOLIC_ERRNAME=y
# end of printk and dmesg options
CONFIG_DEBUG_KERNEL=y
# CONFIG_DEBUG_MISC is not set
-
That's odd. Works on x86_64, I don't know much about Linux on MIPS32 though.
-
ok, checked my mount scripts, I had forgotten to uncomment this line
mount -t debugfs none /sys/kernel/debug
now /sys/kernel/debug/ is populated ;D
meanwhile I am cooking a modern 2023 stage1-2-3-4
-
That would explain it.
So is there anything in the /sys/kernel/debug/tracing/events/syscalls directory?
-
CONFIG_BLOCK=y
Okay, so maybe the syscall wrapper is at fault. What do you get if you run
strace -e ioprio_get prog-in-post-#11
The output should start with a line looking something like
ioprio_get(IOPRIO_WHO_PROCESS, PID) = N (...)
with the three dots corresponding to a C preprocessor macro that reconstructs the I/O priority class and value.
If not, run
strace prog-in-post-#11
and look at the line following
getpid() = PID
because the next line corresponds to whatever syscall the wrapper ended up calling, and with what parameters; the exact ioprio_get line described above.
If both cases yield
ioprio_get(IOPRIO_WHO_PROCESS, PID) = -89 Function not implemented
then the issue is that for some kernel configuration reason, the block/ioprio.o (https://elixir.bootlin.com/linux/latest/source/block/ioprio.c) is not linked to the kernel. (The symbol name in the kernel ELF before compression is sys_ioprio_get with alias __se_sys_ioprio_get, if you happen to have it left over from your kernel build.)
The syscall itself is known and listed, but lacking that object in the kernel, the implementation will just return -89.
(In other words, the list of syscalls the kernel implements is only a subset of the list it knows about; the unimplemented but known ones all return -ENOSYS.)
-
That would explain it.
So is there anything in the /sys/kernel/debug/tracing/events/syscalls directory?
tracing is not yet there, I think something is still missing :-//
-
Okay, so maybe the syscall wrapper is at fault. What do you get if you run
strace -e ioprio_get prog-in-post-#11
# strace -e ioprio_get obj/showioprio
strace: invalid system call `ioprio_get'
strace obj/showioprio
execve("obj/showioprio", ["obj/showioprio"], [/* 27 vars */]) = 0
brk(0) = 0x413000
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x77e6e000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=25936, ...}) = 0
old_mmap(NULL, 25936, PROT_READ, MAP_PRIVATE, 3, 0) = 0x77e67000
close(3) = 0
open("/lib/libc.so.6", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\10\0\1\0\0\0004m\1\0004\0\0\0$"..., 512) = 512
lseek(3, 732, SEEK_SET) = 732
read(3, "\4\0\0\0\20\0\0\0\1\0\0\0GNU\0\0\0\0\0\2\0\0\0\6\0\0\0\t\0\0\0"..., 32) = 32
fstat64(3, {st_mode=S_IFREG|0755, st_size=1579724, ...}) = 0
old_mmap(NULL, 1494176, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x77cd1000
old_mmap(0x77e31000, 40960, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x160000) = 0x77e31000
old_mmap(0x77e3b000, 11424, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x77e3b000
close(3) = 0
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x77e66000
set_thread_area(0x77e6d470) = 0
mprotect(0x77e31000, 32768, PROT_READ) = 0
mprotect(0x411000, 4096, PROT_READ) = 0
mprotect(0x77e6f000, 4096, PROT_READ) = 0
munmap(0x77e67000, 25936) = 0
getpid() = 11360
syscall(0x122, 0x1, 0x2c60, 0, 0x41a030, 0x7fa34d84, 0x77e6fef0, 0x7fa34d84) = -1 ENOSYS (Function not implemented)
write(2, "11360: ioprio_get syscall failed:"..., 6011360: ioprio_get syscall failed: Function not implemented.
) = 60
exit_group(1)
-
block/ioprio.c is not linked to the kernel
kernel-6.02.002-mips-next # ls block/ioprio.o
block/ioprio.o
kernel-6.02.002-mips-next # grep sys_ioprio_get output/kernel.map
80354e4c T __se_sys_ioprio_get
80354e4c T sys_ioprio_get
block/ioprio.c was compiled, and from this (on the running board, so the kernel is responding) it seems to have been linked as well
cat /proc/kallsyms | grep ioprio_get
80354c3c T sys_ioprio_get
80354c3c T __se_sys_ioprio_get
or not? :-//
I cannot understand why my second test reports
syscall(1) = -1 (Function not implemented)
syscall(2) = -1 (Function not implemented)
syscall(3) = -1 (Function not implemented)
syscall(4) = -1 (Function not implemented)
syscall(5) = -1 (Function not implemented)
syscall(6) = -1 (Function not implemented)
syscall(7) = -1 (Function not implemented)
syscall(8) = -1 (Function not implemented)
syscall(9) = -1 (Function not implemented)
syscall(10) = -1 (Function not implemented)
...
syscall(999) = -1 (Function not implemented)
none of the system calls are implemented?!? I can't believe it, because at least the filesystem syscalls work, so they are implemented and seen from userspace.
confused and perplexed :-//
-
syscall(0x122, 0x1, 0x2c60, 0, 0x41a030, 0x7fa34d84, 0x77e6fef0, 0x7fa34d84) = -1 ENOSYS (Function not implemented)
Okay, 0x122 = 290.
On MIPS, using the O32 (https://elixir.bootlin.com/linux/latest/source/arch/mips/kernel/syscalls/syscall_o32.tbl) calling conventions, this is the mknodat() syscall. Using the n32 (https://elixir.bootlin.com/linux/latest/source/arch/mips/kernel/syscalls/syscall_n32.tbl) calling conventions, this is the dup3() syscall. Using the n64 (https://elixir.bootlin.com/linux/latest/source/arch/mips/kernel/syscalls/syscall_n64.tbl) conventions, this is the pwritev() syscall.
Where ioprio_get is syscall 0x122 = 290, is on 32-bit x86 (https://elixir.bootlin.com/linux/latest/source/arch/x86/entry/syscalls/syscall_32.tbl).
In other words, your <sys/syscall.h> includes (<asm/unistd.h>) which is from Linux kernel configured for x86, and includes file <asm/unistd_32.h>, which contains #define __NR_ioprio_get 290. This is absolutely wrong, because it should be 315 (MIPS o32) or 278 (MIPS n32).
Just fixing the syscall numbers does not cut it. You have a development environment which includes kernel-provided userspace headers from 32-bit x86, to be used on MIPS32le. It is almost certainly a bug in the glibc build, where x86 kernel userspace headers are used instead of MIPS32le kernel userspace headers. This means that not only are the syscall numbers wrong, also the kernel structures used by applications try to use the x86 layout, while the kernel will use the MIPS32le layout; and the syscall() function provided by glibc tries to use the x86 syscall convention, obviously failing at it. It explains all the symptoms and issues seen.
-
(In case anyone else cares, if you examine my posts above, you can directly observe my problem-solving method. I've only omitted explicitly using bootlin Elixir Linux kernel cross-referencer (https://elixir.bootlin.com/linux/latest/source) (keyword search in upper right) and plain ol' grep -e 'pattern' -R kernel-source-tree/ to look for kernel files referring to specific tokens or string patterns, and just link to the actual target files found. If you compare to my other posts on any problem involving the Linux kernel, you see the exact same pattern applied there. This is all intentional, because I'm trying to show how to solve this kind of problems, not only this exact problem; and using just this exact problem as an example. My approach may not work, but it is the reason I construct my posts the way I do.)
-
You have a development environment which includes kernel-provided userspace headers from 32-bit x86, to be used on MIPS32le. It is almost certainly a bug in the glibc build
ok, now I have to understand if
- it's a bug of sys-libs/glibc ebuild?
- it's a bug of portage (glibc ebuilds need special support)?
- it's a bug of Catalyst, emerging stage1:sys-libs/glibc (profile bug?)?
- it's a bug of the builder compiling the kernel, and preparing kernel-headers accordingly (profile bug?)?
for sure
1) mipsle profiles are ALL broken since 2015
2) mips2le profiles are ***compatible*** with mips32le but with less performance because less optimized
considering all of this, I will prepare a mips2le branch in parallel, just to see, how it goes ... :-//
-
For sure:
- Kernel headers are massively used when recompiling system libc.
- Kernel headers are managed by Catalyst when recompiling an existing libc to something newer
- Silent failures can cause system libc to not make use of newer features present in the updated kernel headers
- or ... something even worse, since it's an x86 machine cross-compiling for mips
That's why people say native-compiling (even in a Qemu/MIPS sandbox(1)) is safer ... :-//
(1) that's Gentoo way, always native compiling
Whereas, since Qemu/MIPS is slower on a Mac-Mini-i2, I am using an experimental cross-Catalyst engine.
-
You have a development environment which includes kernel-provided userspace headers from 32-bit x86, to be used on MIPS32le. It is almost certainly a bug in the glibc build
ok, now I have to understand if [...]
The problem occurs just before the point that your build machinery runs in the kernel source tree
make mrproper
make headers_install
or
make mrproper
make headers
followed by copying the header files from ./usr/include/ to /usr/include/.
That is done before glibc is built, because you cannot build glibc without the files that generates. Whether it is a separate pass/task before glibc is built, or done as part of the glibc build, I do not know. But it does sound like a bug in Catalyst to me, so that's where I'd start the search.
The problem is that instead of a plain make mrproper, you actually want to start with pristine kernel source tree with the oldest kernel you want to support, and run
make mrproper
cp -f .config-for-the-kernel .config
make oldconfig
make headers
find usr/include/ -type f ! -name '*.h' -delete
and finally install files from ./usr/include/ to /usr/include/. All should be owned by root:root, with directory mode 0755 (drwxr-xr-x) and file mode 0644 (-rw-r--r--).
The .config-for-the-kernel should be configured for the exact target architecture. Right now, your build machinery uses defaults, which causes it to default to x86, or possibly a .config that configures to x86. Since you already have a working kernel, I'd use that .config.
-
The kernel is not built by Catalyst, it's built by mybuilder, which is a classic cross-compiler environment based on this:
macmini-intel kernel-6.02.002-mips-next # cat output/kernel.build.cmd
make ARCH=mips CROSS_COMPILE=mipsel-unknown-linux-gnu-
(this is the actual command passed to compile the kernel)
Catalysts builds the rootfs, recycling a working rootfs, on the top of which re-compiles everything, from glibc, to gcc, to the rest.
So, I think ... if the kernel is not correctly built, then it's a Linux/MIPS bug.
I will try the new stage3/2023-02 as soon as its cooking is completed.
It will take 95 hours.
-
wait, do you mean the sys-lib/glibc ebuild needs to see the kernel source in order to compiler properly?
in this case, my old 2008 ebuilds do not check for the /usr/src/linux folder, which is where the kernel source should be :-//
(in this case, it would be ebuild bug)
-
The kernel is not built by Catalyst
This has nothing to do with building the kernel.
This has everything to do with installing the userspace headers that are required to build glibc.
These headers are generated by running make headers in a suitably configured kernel tree.
This kernel tree can be (and usually is) a different version than the kernel you will eventually run.
These headers expose the kernel-userspace ABI.
Catalysts builds [...] glibc [..] gcc
Then the bug is in Catalyst.
Chapters 4 and 5 in the Linux From Scratch (https://linuxfromscratch.org/lfs/view/stable/index.html) book are useful, because they show the exact commands how this is done "by hand"; specifically Chapter 5.4. Linux API Headers (https://linuxfromscratch.org/lfs/view/stable/chapter05/linux-headers.html).
There is also Cross-Compiled Linux From Scratch (https://trac.clfs.org/wiki/read), but it is "stale"; however, there is a specific guide for 32-bit MIPS (http://www.clfs.org/view/CLFS-3.0.0-SYSVINIT/mips/) based on gcc-4.8.3 and linux-3.14.21, which probably contains most if not all the quirks you need to care about in this kind of cross build to MIPS32.
-
that's what sys-kernel/linux-headers should be supposed to do as requirement of sys-libs/glibc
2023-03-12--18-16---2023-03-12--18-40 - [ sys-kernel/linux-headers ] - success - @2.18/4.1.2
but it somehow fails on a cross-compiling environment.
-
wait, do you mean the sys-lib/glibc ebuild needs to see the kernel source in order to compiler properly?
No, it does not.
I am saying that there is a step before glibc is compiled, where the kernel-userspace API and ABI headers are copied to /usr/include/. This does use Linux kernel sources, but does not build a kernel.
that's what sys-kernel/linux-headers should be supposed to do as requirement of sys-libs/glibc [...] but it somehow fails on a cross-compiling environment.
If you are using gentoo sys-kernel/linux-headers rip-headers.sh (https://gitweb.gentoo.org/proj/toolchain/linux-headers-patches.git/tree/rip-headers.sh), it would explain the problem: it does not configure the kernel, so the headers it extracts are for the default (x86-64) arch.
I took a look at the Linux kernel documentation, exporting kernel headers to userspace (https://www.kernel.org/doc/html/latest/kbuild/headers_install.html), to refrash my addled mind. Oh! No need to mess with .config; setting ARCH suffices.
So, starting at the rm -rf ${dst}, the rip-headers.sh should do just
rm -rf ${dst} ${dst}.tar.xz
mkdir ${dst}
absdst=$(realpath ${dst})
(cd ${src} && make ARCH=mips INSTALL_HDR_PATH=${absdst} headers_install)
Then, everything under ${dst}/include/ is (archived to be) copied to /usr/include/; I believe
tar -cJv ${dst}.tar.xz -C ${dst} include/ && rm -rf ${dst}
should do the trick. Finally, revert the kernel tree back to pristine state,
(cd ${src} && make mrproper)
That would generate a tarball with include/, include/asm, include/asm/unistd.h, and so on. If you need the tarball paths to be usr/include/, use
rm -rf ${dst} ${dst}.tar.xz
mkdir ${dst}
absdst=$(realpath ${dst})
(cd ${src} && make ARCH=mips INSTALL_HDR_PATH=${absdst} headers_install)
mkdir ${dst}/usr
mv ${dst}/include ${dst}/usr/
tar -cJv ${dst}.tar.xz -C ${dst} usr/ && rm -rf ${dst}
(cd ${src} && make mrproper)
instead.
Because the kernel-userspace ABI is stable, you can use the absolutely latest possible kernel that supports the arch for this, even if you compile and install a different, even patched, kernel.
So, there is no need to even play with kernel configs, as setting ARCH suffices. It generates no symlinks or additional files, so no cleanup is needed either.
That installs all three calling conventions (o32, n32, n64), which are selectable in userspace at run time, I believe; at least the syscall numbers do not overlap.
-
If you are using gentoo sys-kernel/linux-headers rip-headers.sh (https://gitweb.gentoo.org/proj/toolchain/linux-headers-patches.git/tree/rip-headers.sh), it would explain the problem: it does not configure the kernel, so the headers it extracts are for the default (x86-64) arch.
Yup, that one.
Now we know it was bad idea (d'oh)
Note Catalyst has been designed to native-compile, so inside a Qemu/MIPS.LE it would have worked perfectly.
What I am doing is Cross-Compiling, so .. I am (ab)using its design.
We can't blame Gentoo, it's my fault.
p.s.
applied your method to a new generation of scripts to cook the new 2023 stage{1,2,3,4} MIPS2/le,O32
with
- sys-libs/glibc-2.34
- sys-kernel/linux-headers-6.2
(supporting the kernel 6.0.2)
p.s.2
I split the new rootfs into parts and recruited a squad of five Mac-Mini-intelCoreDuo2 (with 2Gbyte of ram each!!!), stacked on the top of the other and compiling together for world domination, or something like that ;D
-
new first cooked (temporary) stage4
gcc showprio.c -o showioprio
# ./showioprio
477: Best-effort 4 (default)
#define _GNU_SOURCE
#define _POSIX_C_SOURCE 200809L
#define _BSD_SOURCE
#define _SVID_SOURCE
#include <stdlib.h>
#include <unistd.h>
#include <sys/syscall.h>
#include <sys/resource.h>
#include <linux/sched.h>
#include <sched.h>
#include <string.h>
#include <errno.h>
#include <stdio.h>
int report
(
pid_t p
)
{
errno = 0;
long val = syscall(SYS_ioprio_get, /*IOPRIO_WHO_PROCESS*/ 1, (int) p);
if (errno)
{
if (errno == ESRCH)
{
fprintf(stderr, "%d: No such process.\n", (int) p);
return 0;
}
else
{
fprintf(stderr, "%d: ioprio_get syscall failed: %s.\n", (int) p, strerror(errno));
return -1;
}
}
long c = val >> 13;
if (c == 0)
{
errno = 0;
int n = getpriority(PRIO_PROCESS, p);
if (errno)
{
fprintf(stderr, "%d: getpriority() failed: %s.\n", (int) p, strerror(errno));
return -1;
}
if (n < -20 || n > 19)
{
fprintf(stderr, "%d: getpriority() returned an invalid scheduling priority, %d.\n", (int) p, n);
return -1;
}
int v = (n + 20) / 5; /* Specified as the mapping between scheduling priority and IO priority */
int s = sched_getscheduler(p);
switch (s & 0x3FFFFFFF)
{
case -1:
fprintf(stderr, "%d: sched_getscheduler() failed: %s.\n", (int) p, strerror(errno));
return -1;
case SCHED_FIFO:
case SCHED_RR:
case SCHED_DEADLINE:
printf("%d: Realtime %d (default)\n", (int) p, v);
break;
case SCHED_IDLE:
printf("%d: Idle %d (default)\n", (int) p, v);
break;
default:
printf("%d: Best-effort %d (default)\n", (int) p, v);
break;
}
fflush(stdout);
return 0;
}
else
{
int v = val & 0x1FFF;
if (v < 0 || v > 7)
{
printf("%d: Unknown ioprio 0x%lx\n", (int) p, (unsigned long) val);
fflush(stdout);
return -1;
}
switch (c)
{
case 1: printf("%d: Realtime %d\n", (int) p, v);
break;
case 2: printf("%d: Best-effort %d\n", (int) p, v);
break;
case 3: printf("%d: Idle %d\n", (int) p, v);
break;
default: fprintf(stderr, "%d: Unknown ioprio 0x%lx\n", (int) p, (unsigned long) val);
return -1;
}
fflush(stdout);
return 0;
}
}
int parse_pid
(
char *src,
pid_t *to
)
{
char *end;
long val;
if (!src || !*src)
{
return errno = EINVAL;
}
errno = 0;
end = src;
val = strtol(src, (char * *) (&end), 0);
if (errno)
{
return errno;
}
if (src == end || *end || (long) ((pid_t) val) != val)
{
return errno = EINVAL;
}
if (to)
{
*to = (pid_t) val;
}
return 0;
}
int main
(
int argc,
char *argv[]
)
{
pid_t p;
int arg;
if (argc == 2 && (!strcmp(argv[1], "-h") || !strcmp(argv[1], "--help")))
{
fprintf(stderr, "\n");
fprintf(stderr, "Usage: %s -h | --help\n", argv[0]);
fprintf(stderr, " %s PID [ PID ... ]\n", argv[0]);
fprintf(stderr, "\n");
fprintf(stderr, "This reports the IO class and priority for the specified process or processes.\n");
fprintf(stderr, "\n");
}
if (argc < 2)
{
if (report(getpid()))
{
return EXIT_FAILURE;
}
return EXIT_SUCCESS;
}
for (arg = 1; arg < argc; arg++)
{
if (parse_pid(argv[arg], &p) || p < 1)
{
fprintf(stderr, "%s: Invalid process ID: %s.\n", argv[arg], strerror(errno));
return EXIT_FAILURE;
}
if (report(p))
{
return EXIT_FAILURE;
}
fflush(stdout);
}
return EXIT_SUCCESS;
}
it doesn't look bad.