Skip to content
Snippets Groups Projects
Commit 1181c4cc authored by Petr Baudis's avatar Petr Baudis
Browse files

Initial commit

Complete basic implementation, compiles but not tested yet.
parents
No related branches found
No related tags found
No related merge requests found
.*
*.o
help-in-quotes
compctld
compctl
Makefile 0 → 100644
CFLAGS=-Wall -O3 -std=gnu99
all: compctld compctl
compctld: cgroup.o compctld.o
compctl: cgroup.o compctl.o
compctl.o:: compctl.c help-in-quotes
help-in-quotes: README.client
cat $< | sed -e '0,/^Usage/d; /^Description/,$$d' | \
tail -n +3 | sed -e 's/\(.*\)/"\1\\n"/' >$@
clean:
rm -f compctl compctld compctl.o compctld.o cgroup.o help-in-quotes
README 0 → 100644
**Computations under control** will run a specificed program as
a non-interactive computation process, allowing it to be listed
and regulated by the user of the computer.
To avoid needless use of setuid, it uses a simple client-server
architecture. On boot, compctld daemon is started (as root),
sets up the cgroup infrastructure and accepts requests from clients
for moving a process to cgroup, killing a cgroup'd process and
tweaking the cgroup limits.
The client compctl interface simply queries the server using
a synchronous protocol over a UNIX socket. First, the client
sends a SCM_CREDENTIALS ancilliary message. Then, it follows
with a CRLF-terminated command string and receives a CRLF-terminated
reply string. Connection is closed immediately on breach of protocol.
You can tweak some simple compile-time configuration variables
in file 'common.h'. Build everything using the `make` command.
Then, arrange compctld to be run on boot and put compctl in $PATH
for your users to enjoy.
compctl - Computations under control
Usage
=====
compctl --run COMMAND...
compctl --screen COMMAND...
compctl --usage
compctl --limitmem MAXMEM
compctl --list
compctl --stop PGRP
compctl --stopall
Arguments for running computations:
--run COMMAND run a new computation on foreground
--screen COMMAND run a new computation within a screen instance; all
windows within this screen will share resource limits
Arguments for controlling computations:
--usage show resource usage of all running computations
--limitmem MAXMEM change the maximum memory consumed by all computations
--list list all running computations
--stop PGRP stop a given computation (number as listed in --list)
--stopall stop all running computations
Other options:
--help help message
Description
===========
**Computations under control** will run a specificed program as
a non-interactive computation process, allowing it to be listed
and regulated by the user of the computer.
Examples
========
compctl --run ./satsolver data.sat | tee results.txt
compctl --screen autotest-screen 5
compctl --limitmem 4G
compctl --stop 27134
cgroup.c 0 → 100644
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mount.h>
#include <sys/stat.h>
#include <unistd.h>
#include "cgroup.h"
void (*cgroup_perror)(const char *s) = perror;
static const char cgroupfs[] = "/sys/fs/cgroup/";
int
cgroup_setup(const char *chier, const char *controllers)
{
char chierdir[PATH_MAX];
snprintf(chierdir, sizeof(chierdir), "%s%s", cgroupfs, chier);
if (access(chierdir, F_OK) >= 0)
return 0;
if (mount("none", cgroupfs, "tmpfs", 0, NULL) < 0) {
cgroup_perror(cgroupfs);
return -1;
}
if (mkdir(chierdir, 0777) < 0) {
cgroup_perror(chierdir);
return -1;
}
if (mount("none", chierdir, "cgroup", 0, controllers) < 0) {
cgroup_perror(chierdir);
return -1;
}
return 1;
}
int
cgroup_create(const char *chier, const char *cgroup)
{
char cgroupdir[PATH_MAX];
snprintf(cgroupdir, sizeof(cgroupdir), "%s%s/%s", cgroupfs, chier, cgroup);
if (access(cgroupdir, F_OK) >= 0)
return 0;
if (mkdir(cgroupdir, 0777) < 0) {
cgroup_perror(cgroupdir);
return -1;
}
return 1;
}
int
cgroup_add_task(const char *chier, const char *cgroup, pid_t pid)
{
char tasksfile[PATH_MAX];
snprintf(tasksfile, sizeof(tasksfile), "%s%s/%s/tasks", cgroupfs, chier, cgroup);
FILE *tasks = fopen(tasksfile, "a");
if (!tasks) {
cgroup_perror(tasksfile);
return -1;
}
fprintf(tasks, "%d\n", pid);
fclose(tasks);
return 1;
}
int
cgroup_is_task_in_cgroup(const char *chier, const char *cgroup, pid_t pid)
{
char tasksfile[PATH_MAX];
snprintf(tasksfile, sizeof(tasksfile), "%s%s/%s/tasks", cgroupfs, chier, cgroup);
FILE *tasks = fopen(tasksfile, "r");
if (!tasks) {
cgroup_perror(tasksfile);
return -1;
}
bool exists = false;
char line[128];
while (fgets(line, sizeof(line), tasks)) {
if (atoi(line) == pid) {
exists = true;
break;
}
}
fclose(tasks);
return exists;
}
int
cgroup_task_list(const char *chier, const char *cgroup, pid_t **tasklist)
{
char tasksfile[PATH_MAX];
snprintf(tasksfile, sizeof(tasksfile), "%s%s/%s/tasks", cgroupfs, chier, cgroup);
FILE *tasks = fopen(tasksfile, "r");
if (!tasks) {
cgroup_perror(tasksfile);
return -1;
}
int ntasks = 0;
*tasklist = NULL;
char line[128];
while (fgets(line, sizeof(line), tasks)) {
if (!(ntasks % 32))
*tasklist = realloc(*tasklist, (ntasks + 32) * sizeof(*tasklist));
(*tasklist)[ntasks++] = atoi(line);
}
fclose(tasks);
return ntasks;
}
size_t
cgroup_get_mem_limit(const char *chier, const char *cgroup)
{
char limitfile[PATH_MAX];
snprintf(limitfile, sizeof(limitfile), "%s%s/%s/memory.limit_in_bytes", cgroupfs, chier, cgroup);
FILE *limit = fopen(limitfile, "r");
if (!limit) {
cgroup_perror(limitfile);
return -1;
}
size_t nlimit = 0;
fscanf(limit, "%zu", &nlimit); /* FIXME is this really unsigned? */
fclose(limit);
return nlimit;
}
int
cgroup_set_mem_limit(const char *chier, const char *cgroup, size_t nlimit)
{
char limitfile[PATH_MAX];
snprintf(limitfile, sizeof(limitfile), "%s%s/%s/memory.limit_in_bytes", cgroupfs, chier, cgroup);
FILE *limit = fopen(limitfile, "w");
if (!limit) {
cgroup_perror(limitfile);
return -1;
}
fprintf(limit, "%zu\n", nlimit);
fclose(limit);
return 1;
}
size_t
cgroup_get_mem_usage(const char *chier, const char *cgroup)
{
char usagefile[PATH_MAX];
snprintf(usagefile, sizeof(usagefile), "%s%s/%s/memory.usage_in_bytes", cgroupfs, chier, cgroup);
FILE *usage = fopen(usagefile, "r");
if (!usage) {
cgroup_perror(usagefile);
return -1;
}
size_t nusage = 0;
fscanf(usage, "%zu", &nusage);
fclose(usage);
return nusage;
}
cgroup.h 0 → 100644
/* Tiny cgroup manipulation toolkit. libcgroup is a mess. */
#ifndef CGROUP__H
#define CGROUP__H
#include <stdbool.h>
#include <sys/types.h>
/* In case of errors, this function is called; it is expected to
* report @s and errno contents. By default, it is bound to perror,
* but you can change that if you want e.g. to syslog errors. */
extern void (*cgroup_perror)(const char *s);
/* Setup the cgroups machinery and @chier cgroups hierarchy using
* @controllers. Return 0 if cgroups already set up, 1 on success,
* -1 on failure. */
int cgroup_setup(const char *chier, const char *controllers);
/* Setup a given control group. Return 0 if cgroups already set up,
* 1 on success, -1 on failure. */
int cgroup_create(const char *chier, const char *cgroup);
/* Add a task to a given cgroup. Return 1 on success, -1 on failure. */
int cgroup_add_task(const char *chier, const char *cgroup, pid_t pid);
/* Check if a task is in a given cgroup. Return 0/1, -1 on failure. */
int cgroup_is_task_in_cgroup(const char *chier, const char *cgroup, pid_t pid);
/* Store a list of tasks in a given cgroup to @tasks. Returns number of tasks
* or -1 on failure. */
int cgroup_task_list(const char *chier, const char *cgroup, pid_t **tasks);
/* Get memory limit of a given cgroup. Return (size_t) -1 on failure. */
size_t cgroup_get_mem_limit(const char *chier, const char *cgroup);
/* Set a memory limit of a given cgroup. Return 1 on success, -1 on failure. */
int cgroup_set_mem_limit(const char *chier, const char *cgroup, size_t limit);
/* Get memory usage of a given cgroup. Return (size_t) -1 on failure. */
size_t cgroup_get_mem_usage(const char *chier, const char *cgroup);
#endif
common.h 0 → 100644
/* Definitions common for the client and daemon,
* and runtime configuration definitions. */
#ifndef COMMON__H
#define COMMON__H
/* Compile-time configuration area. */
/* Default memory split policy. Memory is split between
* user and computations by split_ratio, but so that neither
* has reserved less than minfree (in case of too little memory,
* user takes precedence) and user does not have reserved more
* than maxfree. */
size_t static_minfree = 512*1048576UL;
size_t static_maxfree = 2048*1048576UL;
double split_ratio = 0.5;
/* Other common definitions. */
/* See README for the high-level protocol description. */
#define SOCKFILE "/var/run/compctl.sock"
/* CGroup and hierarchy names. */
static const char chier[] = "memory";
static const char cgroup[] = "comp";
#endif
compctl.c 0 → 100644
#define _GNU_SOURCE /* struct ucred */
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <unistd.h>
#include "cgroup.h"
#include "common.h"
FILE *
connectd(void)
{
int s = socket(AF_UNIX, SOCK_STREAM, 0);
struct sockaddr_un sun = { .sun_family = AF_UNIX, .sun_path = SOCKFILE };
if (connect(s, (struct sockaddr *) &sun, sizeof(sun.sun_family) + strlen(sun.sun_path) + 1) < 0) {
perror(SOCKFILE);
exit(EXIT_FAILURE);
}
/* Send message with credentials. */
struct ucred cred = {
.pid = getpid(),
.uid = getuid(),
.gid = getgid(),
};
char cbuf[CMSG_SPACE(sizeof(cred))];
struct msghdr msg = {0};
msg.msg_control = cbuf;
msg.msg_controllen = sizeof(cbuf);
struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg);
cmsg->cmsg_level = SOL_SOCKET;
cmsg->cmsg_type = SCM_CREDENTIALS;
cmsg->cmsg_len = CMSG_LEN(sizeof(cred));
memcpy(CMSG_DATA(cmsg), &cred, sizeof(cred));
sendmsg(s, &msg, 0);
return fdopen(s, "rw");
}
void
help(FILE *f)
{
fputs("compctl - Computations under control\n\n"
#include "help-in-quotes"
"Contact <wizards@kam.mff.cuni.cz> with bug reports and comments.\n", f);
}
int
run(int argc, char *argv[])
{
FILE *f = connectd();
fputs("blessme\r\n", f);
char line[1024];
fgets(line, sizeof(line), f);
fclose(f);
if (line[0] != '1') {
fputs(line, stderr);
return EXIT_FAILURE;
}
char *argvx[argc + 1];
for (int i = 0; i < argc; i++)
argvx[i] = argv[i];
argvx[argc] = NULL;
execvp(argvx[0], argvx);
perror("execvp");
return EXIT_FAILURE;
}
int
screen(int argc, char *argv[])
{
char *argvx[argc + 2];
argvx[0] = "screen";
argvx[1] = "-m";
for (int i = 0; i < argc; i++)
argvx[i + 2] = argv[i];
return run(argc + 2, argvx);
}
void
stop(pid_t pid)
{
FILE *f = connectd();
fprintf(f, "stop %d\r\n", pid);
char line[1024];
fgets(line, sizeof(line), f);
fclose(f);
if (line[0] != '1') {
fputs(line, stderr);
exit(EXIT_FAILURE);
}
}
void
stop_all(void)
{
FILE *f = connectd();
fputs("stopall\r\n", f);
char line[1024];
fgets(line, sizeof(line), f);
fclose(f);
if (line[0] != '1') {
fputs(line, stderr);
exit(EXIT_FAILURE);
}
fputs(line + 2, stdout);
}
void
limit_mem(size_t limit)
{
FILE *f = connectd();
fprintf(f, "limitmem %zu\r\n", limit);
char line[1024];
fgets(line, sizeof(line), f);
fclose(f);
if (line[0] != '1') {
/* TODO: Error message postprocessing. */
fputs(line, stderr);
exit(EXIT_FAILURE);
}
}
void
usage(void)
{
size_t usage = cgroup_get_mem_usage(chier, cgroup);
size_t limit = cgroup_get_mem_limit(chier, cgroup);
printf("Memory usage:\t%zuM / %zuM\n", usage, limit);
}
void
list(void)
{
pid_t *tasks;
int tasks_n = cgroup_task_list(chier, cgroup, &tasks);
if (tasks_n < 0)
exit(EXIT_FAILURE);
for (int i = 0; i < tasks_n; i++) {
/* TODO: Print process details. */
printf("%d\n", tasks[i]);
}
}
int
main(int argc, char *argv[])
{
int optind = 1;
if (argc == optind) {
help(stderr);
return EXIT_FAILURE;
}
while (argc > optind) {
char *cmd = argv[optind++];
if (!strcmp(cmd, "--run")) {
if (argc <= optind) {
fputs("missing arguments for --run\n", stderr);
exit(EXIT_FAILURE);
}
return run(argc - optind, &argv[optind]);
} else if (!strcmp(cmd, "--screen")) {
if (argc <= optind) {
fputs("missing arguments for --screen\n", stderr);
exit(EXIT_FAILURE);
}
return screen(argc - optind, &argv[optind]);
} else if (!strcmp(cmd, "--usage")) {
usage();
} else if (!strcmp(cmd, "--list")) {
list();
} else if (!strcmp(cmd, "--stop")) {
if (argc <= optind) {
fputs("missing argument for --stop\n", stderr);
exit(EXIT_FAILURE);
}
stop(atoi(argv[optind++]));
} else if (!strcmp(cmd, "--stopall")) {
stop_all();
} else if (!strcmp(cmd, "--limitmem")) {
if (argc <= optind) {
fputs("missing argument for --limitmem\n", stderr);
exit(EXIT_FAILURE);
}
limit_mem(atol(argv[optind++]));
} else if (!strcmp(cmd, "--help")) {
help(stdout);
}
}
return EXIT_SUCCESS;
}
#define _GNU_SOURCE /* struct ucred */
#include <assert.h>
#include <errno.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <syslog.h>
#include <unistd.h>
#include "cgroup.h"
#include "common.h"
#define begins_with(s_, a_) (!strncmp(s_, a_, sizeof(a_)))
void
logperror(const char *s)
{
syslog(LOG_ERR, "%s: %s", s, strerror(errno));
}
void
memory_limits(size_t *minuser, size_t *mincomp, size_t *maxcomp, size_t *total)
{
FILE *f = fopen("/proc/meminfo", "r");
char line[1024];
while (fgets(line, sizeof(line), f)) {
if (begins_with("MemTotal:", line)) {
sscanf(line, "MemTotal: %zu", total);
break;
}
}
fclose(f);
*minuser = *total * split_ratio;
if (*minuser < static_minfree)
*minuser = static_minfree;
if (*minuser > static_maxfree)
*minuser = static_maxfree;
*mincomp = static_minfree;
*maxcomp = *total - *minuser;
if (*maxcomp < 0) *maxcomp = 0;
/* maxcomp < mincomp may happen; they are used in different
* settings. */
}
size_t
get_default_mem_limit(void)
{
size_t minuser, mincomp, maxcomp, total;
memory_limits(&minuser, &mincomp, &maxcomp, &total);
return maxcomp;
}
void
cgroup_init(void)
{
if (cgroup_setup(chier, "memory") < 0)
exit(EXIT_FAILURE);
int ret = cgroup_create(chier, cgroup);
if (ret < 0)
exit(EXIT_FAILURE);
if (ret > 0) {
/* CGroup newly created, set limit. */
if (cgroup_set_mem_limit(chier, cgroup, get_default_mem_limit()) < 0)
exit(EXIT_FAILURE);
}
}
int
main(int argc, char *argv[])
{
/* Do this while everyone can still see the error. */
cgroup_init();
pid_t p = fork();
if (p < 0) {
perror("fork");
exit(EXIT_FAILURE);
}
if (p > 0)
exit(EXIT_SUCCESS);
fclose(stderr);
fclose(stdout);
fclose(stdin);
openlog("compctl", LOG_PID, LOG_DAEMON);
cgroup_perror = logperror;
setsid();
int s = socket(AF_UNIX, SOCK_STREAM, 0);
int on = 1; setsockopt(s, SOL_SOCKET, SO_PASSCRED, &on, sizeof(on));
struct sockaddr_un sun = { .sun_family = AF_UNIX, .sun_path = SOCKFILE };
if (bind(s, (struct sockaddr *) &sun, sizeof(sun.sun_family) + strlen(sun.sun_path) + 1) < 0) {
logperror(SOCKFILE);
exit(EXIT_FAILURE);
}
listen(s, 10);
int fd;
while ((fd = accept(s, NULL, NULL)) >= 0) {
/* We handle only a single client at a time. This means
* that it is rather easy to write a script that will DOS
* the daemon, this is just an attack vector we ignore. */
/* TODO: alarm() to wake from stuck clients. */
/* Decode the message with credentials. */
struct msghdr msg;
char *errmsg;
if (recvmsg(fd, &msg, MSG_WAITALL) <= 0) {
errmsg = "recvmsg";
sockerror:
logperror(errmsg);
close(fd);
continue;
}
struct ucred *cred;
struct cmsghdr *cmsg;
cmsg = CMSG_FIRSTHDR(&msg);
if (cmsg == NULL || cmsg->cmsg_len != CMSG_LEN(sizeof(struct ucred))) {
errmsg = "cmsg";
goto sockerror;
}
if (cmsg->cmsg_level != SOL_SOCKET || cmsg->cmsg_type != SCM_CREDENTIALS) {
errmsg = "cmsg designation";
goto sockerror;
}
cred = (struct ucred *) CMSG_DATA(cmsg);
FILE *f = fdopen(fd, "r");
char line[1024];
fgets(line, sizeof(line), f);
size_t linelen = strlen(line);
if (linelen < 2 || strcmp(&line[linelen - 2], "\r\n")) {
syslog(LOG_WARNING, "protocol error (%s)", line);
fclose(f);
continue;
}
line[linelen - 2] = 0;
/* Analyze command */
if (!strcmp("blessme", line)) {
syslog(LOG_INFO, "new computation process %d", cred->pid);
if (cgroup_add_task(chier, cgroup, cred->pid) < 0)
fprintf(f, "0 error: %s\r\n", strerror(errno));
else
fputs("1 blessed\r\n", f);
} else if (begins_with("stop ", line)) {
pid_t pid = atoi(line + sizeof("stop "));
/* Sanity check. */
if (pid < 10 || pid > 32768) {
syslog(LOG_WARNING, "stop: invalid pid (%d)", pid);
fputs("0 invalid pid\r\n", f);
fclose(f);
continue;
}
if (!cgroup_is_task_in_cgroup(chier, cgroup, pid)) {
fputs("0 task not marked as computation\r\n", f);
fclose(f);
continue;
}
syslog(LOG_INFO, "stopping process %d (request by pid %d uid %d)", pid, cred->pid, cred->uid);
kill(pid, SIGTERM);
/* TODO: Grace period and then kill with SIGKILL. */
fputs("1 task stopped\r\n", f);
} else if (!strcmp("stopall", line)) {
pid_t *tasks;
int tasks_n = cgroup_task_list(chier, cgroup, &tasks);
if (tasks_n < 0) {
fprintf(f, "0 error: %s\r\n", strerror(errno));
fclose(f);
continue;
}
for (int i = 0; i < tasks_n; i++) {
syslog(LOG_INFO, "stopping process %d (mass request by pid %d uid %d)", tasks[i], cred->pid, cred->uid);
kill(tasks[i], SIGTERM);
}
/* TODO: Grace period and then kill with SIGKILL. */
fprintf(f, "1 %d tasks stopped\r\n", tasks_n);
free(tasks);
} else if (begins_with("limitmem ", line)) {
size_t limit = atol(line + sizeof("limitmem "));
size_t minuser, mincomp, maxcomp, total;
memory_limits(&minuser, &mincomp, &maxcomp, &total);
/* Sanity check. */
if (limit < 1024 || limit > total) {
syslog(LOG_WARNING, "limitmem: invalid limit (%zu)", limit);
fputs("0 invalid limit value\r\n", f);
fclose(f);
continue;
}
if (limit < mincomp) {
fprintf(f, "-1 at least %zuM must remain available for computations.\r\n", mincomp / 1048576);
fclose(f);
continue;
}
if (total - limit < minuser) {
fprintf(f, "-2 at least %zuM must remain available for users.\r\n", minuser / 1048576);
fclose(f);
continue;
}
syslog(LOG_INFO, "setting limit %zu (request by pid %d uid %d)", limit, cred->pid, cred->uid);
if (cgroup_set_mem_limit(chier, cgroup, limit) < 0)
fprintf(f, "0 error: %s\r\n", strerror(errno));
else
fputs("1 limit set\r\n", f);
} else {
syslog(LOG_WARNING, "invalid command (%s)", line);
}
fclose(f);
}
return EXIT_FAILURE;
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment