From d486f5da0f495d817a19efd077501c25e4eef6fa Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Mon, 27 Mar 2017 11:44:46 +0200 Subject: initial ideas for improving the scheduler API (far from finished) --- src/include/gnunet_scheduler_lib.h | 246 +++++++++++++++++++++++++++- src/util/scheduler.c | 328 ++++++++++++++++++++++++++++++++++++- 2 files changed, 563 insertions(+), 11 deletions(-) diff --git a/src/include/gnunet_scheduler_lib.h b/src/include/gnunet_scheduler_lib.h index 2be1858ce..a7385e31c 100644 --- a/src/include/gnunet_scheduler_lib.h +++ b/src/include/gnunet_scheduler_lib.h @@ -96,6 +96,84 @@ enum GNUNET_SCHEDULER_Reason #include "gnunet_network_lib.h" +/** + * Possible events on FDs, used as a bitmask. + * Modelled after GPollFD. + */ +enum GNUNET_SCHEDULER_EventType +{ + + /** + * No event (useful for timeout). + */ + GNUNET_SCHEDULER_ET_NONE = 0, + + /** + * Data available for reading. + */ + GNUNET_SCHEDULER_ET_IN = 1, + + /** + * Buffer available for writing. + */ + GNUNET_SCHEDULER_ET_OUT = 2, + + /** + * + */ + GNUNET_SCHEDULER_ET_HUP = 4, + + /** + * + */ + GNUNET_SCHEDULER_ET_ERR = 8, + + /** + * + */ + GNUNET_SCHEDULER_ET_PRI = 16, + + /** + * + */ + GNUNET_SCHEDULER_ET_NVAL = 32 + +}; + + +/** + * Information about an event relating to a file descriptor/socket. + */ +struct GNUNET_SCHEDULER_FdInfo +{ + + /** + * GNUnet network socket the event is about, matches @a sock, + * NULL if this is about a file handle or if no network + * handle was given to the scheduler originally. + */ + struct GNUNET_NETWORK_Handle *fd; + + /** + * GNUnet file handle the event is about, matches @a sock, + * NULL if this is about a network socket or if no network + * handle was given to the scheduler originally. + */ + struct GNUNET_DISK_FileHandle *fh; + + /** + * Type of the event that was generated related to @e sock. + */ + enum GNUNET_SCHEDULER_EventType et; + + /** + * Underlying OS handle the event was about. + */ + int sock; + +}; + + /** * Context information passed to each scheduler task. */ @@ -107,32 +185,184 @@ struct GNUNET_SCHEDULER_TaskContext enum GNUNET_SCHEDULER_Reason reason; /** - * Set of file descriptors ready for reading; - * note that additional bits may be set - * that were not in the original request + * Length of the following array. + */ + unsigned int fds_len; + + /** + * Array of length @e fds_len with information about ready FDs. + * Note that we use the same format regardless of the internal + * event loop that was used. The given array should only contain + * information about file descriptors relevant to the current task. + */ + const struct GNUNET_SCHEDULER_FdInfo *fds; + + /** + * Set of file descriptors ready for reading; note that additional + * bits may be set that were not in the original request. + * @deprecated */ const struct GNUNET_NETWORK_FDSet *read_ready; /** - * Set of file descriptors ready for writing; - * note that additional bits may be set - * that were not in the original request. + * Set of file descriptors ready for writing; note that additional + * bits may be set that were not in the original request. + * @deprecated */ const struct GNUNET_NETWORK_FDSet *write_ready; }; +/** + * Function used by event-loop implementations to signal the scheduler + * that a particular @a task is ready due to an event of type @a et. + * + * This function will then queue the task to notify the application + * that the task is ready (with the respective priority). + * + * @param task the task that is ready + * @param et information about why the task is ready + */ +void +GNUNET_SCHEDULER_task_ready (struct GNUNET_SCHEDULER_Task *task, + enum GNUNET_SCHEDULER_EventType et); + + +/** + * Handle to the scheduler's state to be used by the driver. + */ +struct GNUNET_SCHEDULER_Handle; + + +/** + * Function called by the driver to tell the scheduler to run some of + * the tasks that are ready. This function may return even though + * there are tasks left to run just to give other tasks a chance as + * well. If we return #GNUNET_YES, the driver should call this + * function again as soon as possible, while if we return #GNUNET_NO + * it must block until the operating system has more work as the + * scheduler has no more work to do right now. + * + * @param sh scheduler handle that was given to the `loop` + * @return #GNUNET_OK if there are more tasks that are ready, + * and thus we would like to run more (yield to avoid + * blocking other activities for too long) + * #GNUNET_NO if we are done running tasks (yield to block) + * #GNUNET_SYSERR on error + */ +int +GNUNET_SCHEDULER_run_from_driver (struct GNUNET_SCHEDULER_Handle *sh); + + +/** + * API a driver has to implement for + * #GNUNET_SCHEDULER_run_with_driver(). + */ +struct GNUNET_SCHEDULER_Driver +{ + + /** + * Closure to pass to the functions in this struct. + */ + void *cls; + + /** + * Add a @a task to be run if the conditions given + * in @a fdi are satisfied. + * + * @param cls closure + * @param task task to add + * @param fdi conditions to watch for + * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure + * (i.e. @a fdi too high or invalid) + */ + int + (*add)(void *cls, + struct GNUNET_SCHEDULER_Task *task, + struct GNUNET_SCHEDULER_FdInfo *fdi); + + /** + * Delete a @a task from the set of tasks to be run. + * + * @param cls closure + * @param task task to delete + * @param fdi conditions to watch for (must match @e add call) + * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure + * (i.e. @a task or @a fdi do not match prior @e add call) + */ + int + (*del)(void *cls, + struct GNUNET_SCHEDULER_Task *task, + const struct GNUNET_SCHEDULER_FdInfo *fdi); + + /** + * Set time at which we definitively want to get a wakeup call. + * + * @param cls closure + * @param dt time when we want to wake up next + */ + void + (*set_wakeup)(void *cls, + struct GNUNET_TIME_Absolute dt); + + /** + * Event loop's "main" function, to be called from + * #GNUNET_SCHEDULER_run_with_driver() to actually + * launch the loop. + * + * @param cls closure + * @param sh scheduler handle to pass to + * #GNUNET_SCHEDULER_run_from_driver() + * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure + */ + int + (*loop)(void *cls, + struct GNUNET_SCHEDULER_Handle *sh); + +}; + + /** * Signature of the main function of a task. * * @param cls closure - * @param tc context information (why was this task triggered now) */ typedef void (*GNUNET_SCHEDULER_TaskCallback) (void *cls); +/** + * Initialize and run scheduler. This function will return when all + * tasks have completed. On systems with signals, receiving a SIGTERM + * (and other similar signals) will cause #GNUNET_SCHEDULER_shutdown + * to be run after the active task is complete. As a result, SIGTERM + * causes all shutdown tasks to be scheduled with reason + * #GNUNET_SCHEDULER_REASON_SHUTDOWN. (However, tasks added + * afterwards will execute normally!). Note that any particular + * signal will only shut down one scheduler; applications should + * always only create a single scheduler. + * + * @param driver drive to use for the event loop + * @param task task to run first (and immediately) + * @param task_cls closure of @a task + * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure + */ +int +GNUNET_SCHEDULER_run_with_driver (const struct GNUNET_SCHEDULER_Driver *driver, + GNUNET_SCHEDULER_TaskCallback task, + void *task_cls); + + +/** + * Obtain the driver for using select() as the event loop. + * + * @return NULL on error + */ +const struct GNUNET_SCHEDULER_Driver * +GNUNET_SCHEDULER_driver_select (void); + + /** * Signature of the select function used by the scheduler. * #GNUNET_NETWORK_socket_select matches it. @@ -571,7 +801,7 @@ GNUNET_SCHEDULER_add_select (enum GNUNET_SCHEDULER_Priority prio, * Sets the select function to use in the scheduler (scheduler_select). * * @param new_select new select function to use (NULL to reset to default) - * @param new_select_cls closure for 'new_select' + * @param new_select_cls closure for @a new_select */ void GNUNET_SCHEDULER_set_select (GNUNET_SCHEDULER_select new_select, diff --git a/src/util/scheduler.c b/src/util/scheduler.c index 409a0942f..a7b1d8e2a 100644 --- a/src/util/scheduler.c +++ b/src/util/scheduler.c @@ -1,6 +1,6 @@ /* This file is part of GNUnet - Copyright (C) 2009-2016 GNUnet e.V. + Copyright (C) 2009-2017 GNUnet e.V. GNUnet is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published @@ -17,7 +17,6 @@ Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - /** * @file util/scheduler.c * @brief schedule computations using continuation passing style @@ -70,6 +69,35 @@ #define DELAY_THRESHOLD GNUNET_TIME_UNIT_SECONDS +/** + * Argument to be passed from the driver to + * #GNUNET_SCHEDULER_run_from_driver(). Contains the + * scheduler's internal state. + */ +struct GNUNET_SCHEDULER_Handle +{ + /** + * Passed here to avoid constantly allocating/deallocating + * this element, but generally we want to get rid of this. + * @deprecated + */ + struct GNUNET_NETWORK_FDSet *rs; + + /** + * Passed here to avoid constantly allocating/deallocating + * this element, but generally we want to get rid of this. + * @deprecated + */ + struct GNUNET_NETWORK_FDSet *ws; + + /** + * Driver we used for the event loop. + */ + const struct GNUNET_SCHEDULER_Driver *driver; + +}; + + /** * Entry in list of pending tasks. */ @@ -95,6 +123,11 @@ struct GNUNET_SCHEDULER_Task */ void *callback_cls; + /** + * Handle to the scheduler's state. + */ + const struct GNUNET_SCHEDULER_Handle *sh; + /** * Set of file descriptors this task is waiting * for for reading. Once ready, this is updated @@ -110,6 +143,18 @@ struct GNUNET_SCHEDULER_Task */ struct GNUNET_NETWORK_FDSet *write_set; + /** + * Information about which FDs are ready for this task (and why). + */ + const struct GNUNET_SCHEDULER_FdInfo *fds; + + /** + * Storage location used for @e fds if we want to avoid + * a separate malloc() call in the common case that this + * task is only about a single FD. + */ + struct GNUNET_SCHEDULER_FdInfo fdx; + /** * Absolute timeout value for the task, or * #GNUNET_TIME_UNIT_FOREVER_ABS for "no timeout". @@ -123,6 +168,11 @@ struct GNUNET_SCHEDULER_Task struct GNUNET_TIME_Absolute start_time; #endif + /** + * Size of the @e fds array. + */ + unsigned int fds_len; + /** * Why is the task ready? Set after task is added to ready queue. * Initially set to zero. All reasons that have already been @@ -1742,7 +1792,8 @@ GNUNET_SCHEDULER_add_select (enum GNUNET_SCHEDULER_Priority prio, GNUNET_CONTAINER_DLL_insert (pending_head, pending_tail, t); - max_priority_added = GNUNET_MAX (max_priority_added, t->priority); + max_priority_added = GNUNET_MAX (max_priority_added, + t->priority); LOG (GNUNET_ERROR_TYPE_DEBUG, "Adding task %p\n", t); @@ -1750,4 +1801,275 @@ GNUNET_SCHEDULER_add_select (enum GNUNET_SCHEDULER_Priority prio, return t; } + +/** + * Function used by event-loop implementations to signal the scheduler + * that a particular @a task is ready due to an event of type @a et. + * + * This function will then queue the task to notify the application + * that the task is ready (with the respective priority). + * + * @param task the task that is ready, NULL for wake up calls + * @param et information about why the task is ready + */ +void +GNUNET_SCHEDULER_task_ready (struct GNUNET_SCHEDULER_Task *task, + enum GNUNET_SCHEDULER_EventType et) +{ + enum GNUNET_SCHEDULER_Reason reason; + struct GNUNET_TIME_Absolute now; + + now = GNUNET_TIME_absolute_get (); + reason = task->reason; + if (now.abs_value_us >= task->timeout.abs_value_us) + reason |= GNUNET_SCHEDULER_REASON_TIMEOUT; + if ( (0 == (reason & GNUNET_SCHEDULER_REASON_READ_READY)) && + (0 != (GNUNET_SCHEDULER_ET_IN & et)) ) + reason |= GNUNET_SCHEDULER_REASON_READ_READY; + if ( (0 == (reason & GNUNET_SCHEDULER_REASON_WRITE_READY)) && + (0 != (GNUNET_SCHEDULER_ET_OUT & et)) ) + reason |= GNUNET_SCHEDULER_REASON_WRITE_READY; + reason |= GNUNET_SCHEDULER_REASON_PREREQ_DONE; + task->reason = reason; + task->fds = &task->fdx; + task->fdx.et = et; + task->fds_len = 1; + queue_ready_task (task); +} + + +/** + * Function called by the driver to tell the scheduler to run some of + * the tasks that are ready. This function may return even though + * there are tasks left to run just to give other tasks a chance as + * well. If we return #GNUNET_YES, the driver should call this + * function again as soon as possible, while if we return #GNUNET_NO + * it must block until the operating system has more work as the + * scheduler has no more work to do right now. + * + * @param sh scheduler handle that was given to the `loop` + * @return #GNUNET_OK if there are more tasks that are ready, + * and thus we would like to run more (yield to avoid + * blocking other activities for too long) + * #GNUNET_NO if we are done running tasks (yield to block) + * #GNUNET_SYSERR on error + */ +int +GNUNET_SCHEDULER_run_from_driver (struct GNUNET_SCHEDULER_Handle *sh) +{ + enum GNUNET_SCHEDULER_Priority p; + struct GNUNET_SCHEDULER_Task *pos; + struct GNUNET_TIME_Absolute now; + + /* check for tasks that reached the timeout! */ + now = GNUNET_TIME_absolute_get (); + while (NULL != (pos = pending_timeout_head)) + { + if (now.abs_value_us >= pos->timeout.abs_value_us) + pos->reason |= GNUNET_SCHEDULER_REASON_TIMEOUT; + if (0 == pos->reason) + break; + GNUNET_CONTAINER_DLL_remove (pending_timeout_head, + pending_timeout_tail, + pos); + if (pending_timeout_last == pos) + pending_timeout_last = NULL; + queue_ready_task (pos); + } + + if (0 == ready_count) + return GNUNET_NO; + + /* find out which task priority level we are going to + process this time */ + max_priority_added = GNUNET_SCHEDULER_PRIORITY_KEEP; + GNUNET_assert (NULL == ready_head[GNUNET_SCHEDULER_PRIORITY_KEEP]); + /* yes, p>0 is correct, 0 is "KEEP" which should + * always be an empty queue (see assertion)! */ + for (p = GNUNET_SCHEDULER_PRIORITY_COUNT - 1; p > 0; p--) + { + pos = ready_head[p]; + if (NULL != pos) + break; + } + GNUNET_assert (NULL != pos); /* ready_count wrong? */ + + /* process all tasks at this priority level, then yield */ + while (NULL != (pos = ready_head[p])) + { + GNUNET_CONTAINER_DLL_remove (ready_head[p], + ready_tail[p], + pos); + ready_count--; + current_priority = pos->priority; + current_lifeness = pos->lifeness; + active_task = pos; +#if PROFILE_DELAYS + if (GNUNET_TIME_absolute_get_duration (pos->start_time).rel_value_us > + DELAY_THRESHOLD.rel_value_us) + { + LOG (GNUNET_ERROR_TYPE_DEBUG, + "Task %p took %s to be scheduled\n", + pos, + GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_duration (pos->start_time), + GNUNET_YES)); + } +#endif + tc.reason = pos->reason; + GNUNET_NETWORK_fdset_zero (sh->rs); + GNUNET_NETWORK_fdset_zero (sh->ws); + tc.fds_len = pos->fds_len; + tc.fds = pos->fds; + tc.read_ready = (NULL == pos->read_set) ? sh->rs : pos->read_set; + if ( (-1 != pos->read_fd) && + (0 != (pos->reason & GNUNET_SCHEDULER_REASON_READ_READY)) ) + GNUNET_NETWORK_fdset_set_native (sh->rs, + pos->read_fd); + tc.write_ready = (NULL == pos->write_set) ? sh->ws : pos->write_set; + if ((-1 != pos->write_fd) && + (0 != (pos->reason & GNUNET_SCHEDULER_REASON_WRITE_READY))) + GNUNET_NETWORK_fdset_set_native (sh->ws, + pos->write_fd); + LOG (GNUNET_ERROR_TYPE_DEBUG, + "Running task: %p\n", + pos); + pos->callback (pos->callback_cls); + active_task = NULL; + dump_backtrace (pos); + destroy_task (pos); + tasks_run++; + } + if (0 == ready_count) + return GNUNET_NO; + return GNUNET_OK; +} + + +/** + * Initialize and run scheduler. This function will return when all + * tasks have completed. On systems with signals, receiving a SIGTERM + * (and other similar signals) will cause #GNUNET_SCHEDULER_shutdown + * to be run after the active task is complete. As a result, SIGTERM + * causes all shutdown tasks to be scheduled with reason + * #GNUNET_SCHEDULER_REASON_SHUTDOWN. (However, tasks added + * afterwards will execute normally!). Note that any particular + * signal will only shut down one scheduler; applications should + * always only create a single scheduler. + * + * @param driver drive to use for the event loop + * @param task task to run first (and immediately) + * @param task_cls closure of @a task + * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure + */ +int +GNUNET_SCHEDULER_run_with_driver (const struct GNUNET_SCHEDULER_Driver *driver, + GNUNET_SCHEDULER_TaskCallback task, + void *task_cls) +{ + int ret; + struct GNUNET_SIGNAL_Context *shc_int; + struct GNUNET_SIGNAL_Context *shc_term; +#if (SIGTERM != GNUNET_TERM_SIG) + struct GNUNET_SIGNAL_Context *shc_gterm; +#endif +#ifndef MINGW + struct GNUNET_SIGNAL_Context *shc_quit; + struct GNUNET_SIGNAL_Context *shc_hup; + struct GNUNET_SIGNAL_Context *shc_pipe; +#endif + struct GNUNET_SCHEDULER_Task tsk; + const struct GNUNET_DISK_FileHandle *pr; + struct GNUNET_SCHEDULER_Handle sh; + + /* general set-up */ + GNUNET_assert (NULL == active_task); + GNUNET_assert (NULL == shutdown_pipe_handle); + shutdown_pipe_handle = GNUNET_DISK_pipe (GNUNET_NO, + GNUNET_NO, + GNUNET_NO, + GNUNET_NO); + GNUNET_assert (NULL != shutdown_pipe_handle); + pr = GNUNET_DISK_pipe_handle (shutdown_pipe_handle, + GNUNET_DISK_PIPE_END_READ); + GNUNET_assert (NULL != pr); + my_pid = getpid (); + + /* install signal handlers */ + LOG (GNUNET_ERROR_TYPE_DEBUG, + "Registering signal handlers\n"); + shc_int = GNUNET_SIGNAL_handler_install (SIGINT, + &sighandler_shutdown); + shc_term = GNUNET_SIGNAL_handler_install (SIGTERM, + &sighandler_shutdown); +#if (SIGTERM != GNUNET_TERM_SIG) + shc_gterm = GNUNET_SIGNAL_handler_install (GNUNET_TERM_SIG, + &sighandler_shutdown); +#endif +#ifndef MINGW + shc_pipe = GNUNET_SIGNAL_handler_install (SIGPIPE, + &sighandler_pipe); + shc_quit = GNUNET_SIGNAL_handler_install (SIGQUIT, + &sighandler_shutdown); + shc_hup = GNUNET_SIGNAL_handler_install (SIGHUP, + &sighandler_shutdown); +#endif + + /* Setup initial tasks */ + current_priority = GNUNET_SCHEDULER_PRIORITY_DEFAULT; + current_lifeness = GNUNET_YES; + memset (&tsk, + 0, + sizeof (tsk)); + active_task = &tsk; + tsk.sh = &sh; + GNUNET_SCHEDULER_add_with_reason_and_priority (task, + task_cls, + GNUNET_SCHEDULER_REASON_STARTUP, + GNUNET_SCHEDULER_PRIORITY_DEFAULT); + GNUNET_SCHEDULER_add_now_with_lifeness (GNUNET_NO, + &GNUNET_OS_install_parent_control_handler, + NULL); + active_task = NULL; + driver->set_wakeup (driver->cls, + GNUNET_TIME_absolute_get ()); + + /* begin main event loop */ + sh.rs = GNUNET_NETWORK_fdset_create (); + sh.ws = GNUNET_NETWORK_fdset_create (); + sh.driver = driver; + ret = driver->loop (driver->cls, + &sh); + GNUNET_NETWORK_fdset_destroy (sh.rs); + GNUNET_NETWORK_fdset_destroy (sh.ws); + + /* uninstall signal handlers */ + GNUNET_SIGNAL_handler_uninstall (shc_int); + GNUNET_SIGNAL_handler_uninstall (shc_term); +#if (SIGTERM != GNUNET_TERM_SIG) + GNUNET_SIGNAL_handler_uninstall (shc_gterm); +#endif +#ifndef MINGW + GNUNET_SIGNAL_handler_uninstall (shc_pipe); + GNUNET_SIGNAL_handler_uninstall (shc_quit); + GNUNET_SIGNAL_handler_uninstall (shc_hup); +#endif + GNUNET_DISK_pipe_close (shutdown_pipe_handle); + shutdown_pipe_handle = NULL; + return ret; +} + + +/** + * Obtain the driver for using select() as the event loop. + * + * @return NULL on error + */ +const struct GNUNET_SCHEDULER_Driver * +GNUNET_SCHEDULER_driver_select () +{ + GNUNET_break (0); // not implemented + return NULL; +} + + /* end of scheduler.c */ -- cgit v1.2.3