rewrite application stuff in the kernel to support multitasking

applications/goldman/makefile

@@ -0,0 +1,12 @@
+SOURCES = \
+  main.cpp
+
+build/%.cpp.o: source/%.cpp
+	@mkdir -p $(@D)
+	$(HILBERT_CC) -c $^ -o $@
+
+build/goldman.elf: $(SOURCES:%=build/%.o)
+	$(HILBERT_CC) $^ -o $@
+
+clean:
+	rm -rf build

applications/goldman/source/main.cpp

@@ -0,0 +1,4 @@
+int main(int, char **) {
+  while (1)
+    ;
+}

applications/init/makefile

@@ -6,7 +6,7 @@ build/%.cpp.o: source/%.cpp
 	$(HILBERT_CC) -c $^ -o $@
 
 build/init.elf: $(SOURCES:%=build/%.o)
-	$(HILBERT_CC) $^ -ldaguerre -o $@
+	$(HILBERT_CC) $^ -o $@
 
 clean:
 	rm -rf build

applications/init/source/main.cpp

@@ -1,121 +1,19 @@
-#include <daguerre.hpp>
+#include <euler/start_process.hpp>
-
-static daguerre::hilbert_color transparent_color;
-
-void alpha_overlay(
-  daguerre::hilbert_color &dest, const daguerre::hilbert_color &src) {
-  if (src != transparent_color)
-    dest = src;
-}
 
 int main(int, char **) {
 
-  daguerre::default_overlay(
+  __euler_process_handle dummy;
-    transparent_color, (daguerre::rgb24){.r = 255, .g = 0, .b = 255});
 
-  auto raw_framebuffer = daguerre::get_hilbert_framebuffer();
+  euler::start_process wm_process("/bin/compositor");
-  const unsigned fbw = raw_framebuffer.width;
+  wm_process.add_env_variable("ARGC", "1");
-  const unsigned fbh = raw_framebuffer.height;
+  wm_process.add_env_variable("ARGV0", "/bin/compositor");
+  wm_process.start(dummy);
 
-  daguerre::image<daguerre::rgb24> raw_burden;
+  euler::start_process hello_process("/bin/hello");
-  daguerre::try_load_ppm("/assets/burden.ppm", raw_burden);
+  hello_process.add_env_variable("ARGC", "1");
+  hello_process.add_env_variable("ARGV0", "/bin/hello");
+  hello_process.start(dummy);
 
-  daguerre::image<daguerre::rgb24> burden_stretch(fbw, fbh);
+  return 0;
-  daguerre::image<daguerre::rgb24> burden_stretch_inverted(fbw, fbh);
-  for (unsigned y = 0; y < fbh; ++y)
-    for (unsigned x = 0; x < fbw; ++x) {
-      daguerre::rgb24 color = raw_burden.get(
-        x * raw_burden.width / fbw, y * raw_burden.height / fbh);
-      burden_stretch.set(x, y, color);
-      burden_stretch_inverted.set(x, y, {
-        .r = (uint8_t)(255 - color.r),
-        .g = (uint8_t)(255 - color.g),
-        .b = (uint8_t)(255 - color.b)});
-    }
-
-  daguerre::image<daguerre::rgb24> pointer;
-  daguerre::try_load_ppm("/assets/pointer.ppm", pointer);
-
-  daguerre::fixed_bitmap_font<bool> terminus;
-  daguerre::try_load_psf("/assets/terminus-bold-18x10.psf", terminus);
-
-  terminus.overlay_text<>(burden_stretch, 0, 0, "this is a test");
-  terminus.overlay_text<>(burden_stretch_inverted, 0, 0, "tset a si siht");
-
-  daguerre::image<daguerre::hilbert_color>
-    burden_stretch_hilbert(burden_stretch);
-
-  daguerre::image<daguerre::hilbert_color>
-    burden_stretch_inverted_hilbert(burden_stretch_inverted);
-
-  daguerre::image<daguerre::hilbert_color> pointer_hilbert(pointer);
-
-  daguerre::image<daguerre::hilbert_color> double_buffer(
-    fbw + pointer.width - 1, fbh + pointer.height - 1);
-
-  int32_t mouse_x = fbw / 2;
-  int32_t mouse_y = fbh / 2;
-  bool was_left_mouse_down = false;
-
-  bool should_draw = true;
-
-  while (1) {
-
-    if (should_draw) {
-      double_buffer.overlay_from<>(
-        burden_stretch_hilbert, 0, 0, 0, 0, fbw, fbh);
-      double_buffer.overlay_from<daguerre::hilbert_color, alpha_overlay>(
-        pointer_hilbert, mouse_x, mouse_y,
-        0, 0, pointer.width, pointer.height);
-      raw_framebuffer.overlay_from<>(double_buffer, 0, 0, 0, 0, fbw, fbh);
-      should_draw = false;
-    }
-
-    __euler_mouse_buttons mouse_buttons;
-    int16_t mouse_change_x, mouse_change_y;
-    uint32_t key_packet;
-    __euler_input_packet_type packet_type = __euler_get_input_packet(
-      mouse_buttons, mouse_change_x, mouse_change_y, key_packet);
-
-    bool should_invert = false;
-
-    if (packet_type == __EULER_IPT_MOUSE) {
-
-      if (mouse_change_x != 0 || mouse_change_y != 0) {
-
-        mouse_x += mouse_change_x;
-        mouse_y += mouse_change_y;
-
-        if (mouse_x < 0)
-          mouse_x = 0;
-        else if ((unsigned)mouse_x >= fbw)
-          mouse_x = fbw - 1;
-
-        if (mouse_y < 0)
-          mouse_y = 0;
-        else if ((unsigned)mouse_y >= fbh)
-          mouse_y = fbh - 1;
-
-        should_draw = true;
-
-      }
-
-      if (!was_left_mouse_down && (mouse_buttons & __EULER_MB_LEFT))
-        should_invert = true;
-
-      was_left_mouse_down = mouse_buttons & __EULER_MB_LEFT;
-
-    }
-
-    else if (packet_type == __EULER_IPT_KEYBOARD)
-      if ((key_packet & 0x0400ff) == 0x00005a)
-        should_invert = true;
-
-    if (should_invert) {
-      daguerre::swap(burden_stretch_hilbert, burden_stretch_inverted_hilbert);
-      should_draw = true;
-    }
-
-  }
 
 }

documentation/compositor.txt

@@ -1,14 +1,25 @@
 compositors listen on the socket id "hilbert.compositor".
 
+when a window is opened by an application, that window can only be referred to
+on that stream. the opaque value given in the "window opened" message refers to
+that window in future messages on that stream. it is guaranteed to be distinct
+for different windows on the same stream, and in no way guaranteed to be
+distinct for different windows on different streams. the window is bound
+just to the stream, not to the application. if the stream where a window
+was created is gifted to a new process, the new process has complete control
+over the window, and the compositor does not need to be informed.
+
 data types:
 
-  color24:
+  color:
-    byte: red
+    opaque dword (result of encode color system call).
-    byte: green
+    from c++, use __euler_encode_color in euler/syscall.hpp.
-    byte: blue
 
-  color24 rectangle:
+  color rectangle:
-    multiple color24's, top to bottom by row, left to right within row
+    multiple hilbert colors, top to bottom by row, left to right within row
+
+  window:
+    opaque word (given in "window opened" message after "open window" message)
 
 messages from applications to compositor:
 
@@ -19,8 +30,15 @@ messages from applications to compositor:
 
   update window region:
     byte: 0x01
+    window: the window
     dword: start x
     dword: start y
     dword: width
     dword: height
-    color24 rectangle: the data
+    color rectangle: the data
+
+messages from compositor to application:
+
+  window opened:
+    byte: 0x00
+    window: the window

documentation/kernel-interfaces/app-entry.txt

@@ -0,0 +1,3 @@
+on entry, the stack is set up, and all registers other than rsp are set to 0.
+the ARGC environment variable holds the number of arguments to main.
+the ARGV0, ARGV1, ARGV2, etc environment variables hold those arguments.

documentation/kernel-interfaces/syscalls.txt

@@ -1,15 +1,6 @@
-on application entry:
+rax, rdi, rsi, rdx are in/out paramters.
-  there is a 1MiB - 8KiB area mapped writable and not
+rbx, rbp, rsp, r12-r15 are preserved.
-  executable with guard pages on either side, and rsp is
+rcx, rflags, r8-r11 are clobbered.
-  set to the top of that. all other registers are set to 0.
-
-the ARGC environment variable holds the number of arguments to main.
-the ARGV0, ARGV1, ARGV2, etc environment variables hold those arguments.
-
-for all system calls:
-  rax, rdi, rsi, rdx are in/out paramters.
-  rbx, rbp, rsp, r12-r15 are preserved.
-  rcx, rflags, r8-r11 are clobbered.
 
 interrupts (including the timer!) are disabled during system calls.
 
@@ -105,8 +96,6 @@ connect to socket:
   rsi in: id string length
   rax out: stream result
   rdi out: stream handle (if rax = 0)
-  if the listener is closed while this syscall is blocked, rax is
-  set to "socket id not in use" and not "socket listener closed"
 
 close stream:
   rax in: 11
@@ -165,6 +154,7 @@ start process:
     qword: stream handle here
     qword: new stream handle in child
       new handle must be < 65536
+  any gifted streams must not have threads waiting to read from our end
 
 end this process:
   rax in: 17
@@ -176,3 +166,10 @@ set stream length:
   rdi in: stream handle
   rsi in: new length
   rax out: stream result
+
+get other end process handle:
+  returns "other end closed" for files
+  rax in: 19
+  rdi in: stream handle
+  rax out: stream result
+  rdi out: process handle (if rax = success)

documentation/kernel/applications.txt

@@ -0,0 +1,41 @@
+globally there are:
+  an id-list of processes
+  an id-list of threads
+  a queue of paused threads
+  a single running thread
+
+a paused thread is a thread that can run, but isn't.
+a thread that is not running and is not paused is in a blocking syscall.
+
+each process has:
+  a process id
+  a list of threads
+  a list of open file streams
+  a list of open socket stream ends
+  a list of running socket listeners
+  a list of environment variables (pairs of strings)
+
+a thread has:
+  an owning process
+  optionally:
+    a stream end it is waiting to read from
+    a socket listener it is waiting to accept a connection from
+    a socket listener it is waiting to connect to
+  if not running:
+    saved cpu state
+
+a socket has:
+  two queues
+
+an open socket stream end has:
+  a list of threads waiting to read from it
+  the socket
+  its input queue
+  its output queue
+  if the other side is open:
+    the other process
+    the other stream end
+
+a running socket listener has:
+  a list of threads waiting to accept a connection from it
+  a list of threads waiting to connect to it

documentation/kernel/memory.txt

@@ -2,9 +2,10 @@ only the first 32GiB of physical memory are considered. this can be changed
 with pram_pages in paging.cpp. vram layout is as follows:
 
 0x0000.0000.0000 - 0x0000.0000.0fff: always unmapped
-0x0000.0000.1000 - 0x0000.001f.efff: user stack
+0x0000.0000.1000 - 0x003f.ffff.ffff: available to user process
-0x0000.001f.f000 - 0x0000.001f.ffff: always unmapped
+0x0040.0000.0000 - 0x007f.ffff.ffff: each 16MB:
-0x0000.0020.0000 - 0x007f.ffff.ffff: available to user process
+       0x00.0000 -        0x00.0fff: always unmapped
+       0x00.1000 -        0xff.ffff: available for user thread stack
 0x0080.0000.0000 - 0xffff.bfff.ffff: always unmapped
 0xffff.c000.0000 - 0xffff.ffdf.ffff: available to kernel
 0xffff.ffe0.0000 - 0xffff.ffe0.0fff: always unmapped

euler/include/euler/start_process.hpp

@@ -0,0 +1,30 @@
+#pragma once
+
+#include <euler/syscall.hpp>
+#include <vector>
+
+namespace euler {
+
+  struct start_process {
+
+  private:
+    const char *path;
+    uint64_t path_len;
+    std::vector<__euler_env_var_spec> env_var_specs;
+    std::vector<__euler_gift_stream_spec> gift_stream_specs;
+
+  public:
+    //path needs to stay valid through any call to start
+    start_process(const char *path);
+
+    //name and value need to stay valid through any call to start
+    void add_env_variable(const char *name, const char *value);
+
+    void gift_stream(
+      __euler_stream_handle to_gifter, __euler_stream_handle to_giftee);
+
+    __euler_stream_result start(__euler_process_handle &handle_out);
+
+  };
+
+}

euler/include/euler/syscall.hpp

@@ -27,6 +27,7 @@ enum __euler_seek_from : uint8_t {
 };
 
 typedef uint64_t __euler_stream_handle;
+typedef uint64_t __euler_process_handle;
 
 extern "C" __euler_stream_result __euler_open_file(
   const char *path, uint64_t path_len, __euler_stream_handle &handle_out,
@@ -72,3 +73,30 @@ enum __euler_input_packet_type : uint8_t {
 extern "C" __euler_input_packet_type __euler_get_input_packet(
   __euler_mouse_buttons &buttons_out, int16_t &x_change_out,
   int16_t &y_change_out, uint32_t &keyboard_packet_out);
+
+struct [[gnu::packed]] __euler_env_var_spec {
+  uint64_t name_len;
+  const char *name;
+  uint64_t value_len;
+  const char *value;
+};
+
+struct [[gnu::packed]] __euler_gift_stream_spec {
+  __euler_stream_handle stream_handle_to_gifter;
+  __euler_stream_handle stream_handle_to_giftee;
+};
+
+struct [[gnu::packed]] __euler_process_start_info {
+  uint64_t file_path_length;
+  const char *file_path;
+  uint64_t env_var_count;
+  const __euler_env_var_spec *env_vars;
+  uint64_t gift_stream_count;
+  const __euler_gift_stream_spec *gift_streams;
+};
+
+extern "C" __euler_stream_result __euler_start_process(
+  const __euler_process_start_info &info, __euler_process_handle &handle_out);
+
+extern "C" __euler_stream_result __euler_get_other_end_process_handle(
+  __euler_stream_handle handle_in, __euler_process_handle &handle_out);

euler/include/type_traits

@@ -0,0 +1,23 @@
+#pragma once
+
+namespace std {
+
+  template <class t>
+  struct remove_reference {
+    typedef t type;
+  };
+
+  template <class t>
+  struct remove_reference<t &> {
+    typedef t type;
+  };
+
+  template <class t>
+  struct remove_reference<t &&> {
+    typedef t type;
+  };
+
+  template <class t>
+  using remove_reference_t = typename remove_reference<t>::type;
+
+}

euler/include/utility

@@ -0,0 +1,12 @@
+#pragma once
+
+#include <type_traits>
+
+namespace std {
+
+  template <class t>
+  constexpr std::remove_reference_t<t> &&move(t &&x) {
+    return static_cast<std::remove_reference_t<t> &&>(x);
+  }
+
+}

euler/include/vector

@@ -0,0 +1,62 @@
+#pragma once
+
+#include <stddef.h>
+#include <utility>
+
+namespace std {
+
+  template <class t>
+  class vector {
+
+    t *buffer;
+    size_t buffer_length;//always positive
+    size_t count;
+
+  public:
+    vector() : buffer(new t[16]), buffer_length(16), count(0) {}
+
+    ~vector() {
+      delete[] buffer;
+    }
+
+    t &operator[](size_t pos) {
+      return buffer[pos];
+    }
+
+    const t &operator[](size_t pos) const {
+      return buffer[pos];
+    }
+
+    t *data() {
+      return buffer;
+    }
+
+    const t *data() const {
+      return buffer;
+    }
+
+    size_t size() const {
+      return count;
+    }
+
+    void reserve(size_t new_length) {
+      if (new_length <= buffer_length)
+        return;
+      t *new_buffer = new t[new_length];
+      for (size_t i = 0; i < count; ++i)
+        new_buffer[i] = std::move(buffer[i]);
+      delete[] buffer;
+      buffer = new_buffer;
+      buffer_length = new_length;
+    }
+
+    void push_back(t &&value) {
+      if (count == buffer_length)
+        reserve(count * 2);
+      buffer[count++] = std::move(value);
+    }
+
+    //TODO
+  };
+
+}

euler/makefile

@@ -1,6 +1,7 @@
 LIBSTDCPP_SOURCES = euler/stream.cpp strings/strlen.cpp euler/syscall.asm \
   euler/entry.cpp io/fopen.cpp euler/gcc.asm memory/delete.cpp euler/heap.cpp \
-  memory/new.cpp io/fclose.cpp io/fread.cpp strings/memcpy.cpp io/fseek.cpp
+  memory/new.cpp io/fclose.cpp io/fread.cpp strings/memcpy.cpp io/fseek.cpp \
+  euler/start_process.cpp
 
 clean:
 	rm -rf build

euler/source/euler/start_process.cpp

@@ -0,0 +1,35 @@
+#include <euler/start_process.hpp>
+#include <cstring>
+
+namespace euler {
+
+  start_process::start_process(const char *path)
+  : path(path), path_len(std::strlen(path)) {}
+
+  void start_process::add_env_variable(const char *name, const char *value) {
+    env_var_specs.push_back({
+      .name_len = std::strlen(name), .name = name,
+      .value_len = std::strlen(value), .value = value });
+  }
+
+  void start_process::gift_stream(
+    __euler_stream_handle to_gifter, __euler_stream_handle to_giftee) {
+    gift_stream_specs.push_back({
+      .stream_handle_to_gifter = to_gifter,
+      .stream_handle_to_giftee = to_giftee });
+  }
+
+  __euler_stream_result start_process::start(
+    __euler_process_handle &handle_out) {
+    __euler_process_start_info info = {
+      .file_path_length = path_len,
+      .file_path = path,
+      .env_var_count = env_var_specs.size(),
+      .env_vars = env_var_specs.data(),
+      .gift_stream_count = gift_stream_specs.size(),
+      .gift_streams = gift_stream_specs.data(),
+    };
+    return __euler_start_process(info, handle_out);
+  }
+
+}

euler/source/euler/syscall.asm

@@ -12,6 +12,8 @@ global __euler_read_from_stream
 global __euler_get_framebuffer
 global __euler_encode_color
 global __euler_get_input_packet
+global __euler_start_process
+global __euler_get_other_end_process_handle
 
 section .text
 
@@ -124,3 +126,19 @@ __euler_get_input_packet:
   mov byte [rdx], al
   mov al, 1
   ret
+
+__euler_start_process:
+  push rsi
+  mov rax, 16
+  syscall
+  pop rsi
+  mov qword [rsi], rdi
+  ret
+
+__euler_get_other_end_process_handle:
+  push rsi
+  mov rax, 19
+  syscall
+  pop rsi
+  mov qword [rsi], rdi
+  ret

kernel/include/hilbert/kernel/app-memory.hpp

@@ -0,0 +1,61 @@
+#pragma once
+
+#include <stdint.h>
+
+//in the lower half, memory is only ever mapped below 0x0080.0000.0000. this is
+//one p3's worth. the p4 and p3 in app_memory are always real, but p3 may have
+//zero entries indicating p2's that aren't needed. similarly, the p2's may have
+//zero entries indicating p1's that aren't needed.
+
+namespace hilbert::kernel {
+
+  class app_memory {
+
+    typedef uint64_t *v_page_table;
+
+    v_page_table p4;
+    v_page_table p3;
+    v_page_table p2s[512];
+    v_page_table *p1s[512];
+
+    bool **pram_pages_to_free_on_exit[512];
+
+  public:
+    uint64_t p4_paddr;
+
+    app_memory();
+    ~app_memory();
+
+    //vaddr and paddr must be page aligned.
+    //vaddr must be < 0x0080.0000.0000.
+    void map_page(
+      uint64_t vaddr, uint64_t paddr, bool write,
+      bool execute, bool free_pram_on_exit);
+
+    //also frees the pram if marked in pram_pages_to_free_on_exit
+    void unmap_page(uint64_t vaddr);
+
+    bool valid_to_read(
+      uint64_t vaddr_start, uint64_t vaddr_end, bool and_write) const;
+
+    inline bool valid_to_read(
+      const void *vaddr_start, const void *vaddr_end, bool and_write) const {
+      return valid_to_read(
+        (uint64_t)vaddr_start, (uint64_t)vaddr_end, and_write);
+    }
+
+    //pages are together; returns start of first page.
+    //only looks in range [0x0000.0000.1000, 0x0040.0000.0000)
+    uint64_t get_free_vaddr_pages(uint64_t count);
+
+    //returns top of stack
+    uint64_t map_new_stack();
+    //also frees pram pages
+    void unmap_stack(uint64_t top);
+
+    //in lower half
+    uint64_t count_mapped_vram_pages() const;
+
+  };
+
+}

kernel/include/hilbert/kernel/application.hpp

@@ -1,23 +1,14 @@
 #pragma once
 
+#include <hilbert/kernel/app-memory.hpp>
+#include <hilbert/kernel/utility.hpp>
 #include <hilbert/kernel/vfile.hpp>
 #include <stdint.h>
 
-//TODO: end application, threading.
-
 namespace hilbert::kernel::application {
 
-  class process;
-  class thread;
-
-  enum class thread_state {
-    running,
-    paused,
-    waiting
-  };
-
   enum class stream_result {
-    success,
+    success = 0,
     bad_handle,
     io_error,
     out_of_bounds,
@@ -26,7 +17,7 @@ namespace hilbert::kernel::application {
     not_an_executable,
     not_writable,
     not_seekable,
-    socket_id_already_used,
+    socket_id_already_in_use,
     socket_id_not_in_use,
     socket_listener_closed,
     other_end_closed,
@@ -34,83 +25,6 @@ namespace hilbert::kernel::application {
     not_sized
   };
 
-  enum class seek_origin {
-    beginning,
-    end,
-    current_position
-  };
-
-  class stream {
-  public:
-    virtual ~stream() {}
-    virtual stream_result seek(seek_origin origin, int64_t offset) = 0;
-    virtual stream_result read(uint64_t count, void *into) = 0;
-    virtual stream_result write(uint64_t count, const void *from) = 0;
-    virtual stream_result get_length(uint64_t &out) = 0;
-    virtual stream_result set_length(uint64_t to) = 0;
-  };
-
-  class vfile_stream : public stream {
-
-  private:
-    vfile::vfile file;
-    uint64_t offset;
-
-  public:
-    vfile_stream(vfile::vfile &&file);
-    virtual stream_result seek(seek_origin origin, int64_t offset) override;
-    virtual stream_result read(uint64_t count, void *into) override;
-    virtual stream_result write(uint64_t count, const void *from) override;
-    virtual stream_result get_length(uint64_t &out) override;
-    virtual stream_result set_length(uint64_t to) override;
-
-  };
-
-  struct socket {
-    utility::queue<thread *> process_a_threads_waiting_to_read;
-    utility::queue<thread *> process_b_threads_waiting_to_read;
-    utility::queue<uint8_t> a_to_b;
-    utility::queue<uint8_t> b_to_a;
-    bool a_closed;
-    bool b_closed;
-  };
-
-  class socket_stream : public stream {
-
-  private:
-    socket *sock;
-    bool are_we_b;
-
-    utility::queue<thread *> &our_threads_waiting_to_read;
-    utility::queue<thread *> &their_threads_waiting_to_read;
-    utility::queue<uint8_t> &them_to_us;
-    utility::queue<uint8_t> &us_to_them;
-    bool &them_closed;
-    bool &us_closed;
-
-  public:
-    socket_stream(socket *sock, bool are_we_b);
-    ~socket_stream();
-    virtual stream_result seek(seek_origin origin, int64_t offset) override;
-    virtual stream_result read(uint64_t count, void *into) override;
-    virtual stream_result write(uint64_t count, const void *from) override;
-    virtual stream_result get_length(uint64_t &out) override;
-    virtual stream_result set_length(uint64_t to) override;
-
-  };
-
-  struct string_pair {
-    utility::string a;
-    utility::string b;
-  };
-
-  struct socket_listener {
-    utility::string id;
-    utility::queue<thread *> waiting_to_accept_connection;
-    utility::queue<thread *> waiting_to_connect;
-    bool is_listening;
-  };
-
   struct [[gnu::packed]] cpu_state {
 
     uint64_t rax;
@@ -141,80 +55,180 @@ namespace hilbert::kernel::application {
 
   };
 
-  struct thread {
+  class process;
-
+  class thread;
-    //propogated to process on destruction if this is the last thread.
+  class file_stream;
-    int exit_code;
+  class socket;
-    ~thread();
+  class socket_stream_end;
-
+  class socket_listener;
-    process *the_process;
-    thread_state state;
-    //only valid if paused or waiting
-    cpu_state cpu;
-
-    stream *just_connected_to;
-    stream *just_accepted;
 
+  struct generic_stream_ptr {
+    bool is_socket;
+    union {
+      file_stream *as_file_stream;
+      socket_stream_end *as_socket_stream;
+    };
+    bool is_null() {
+      return !is_socket && as_file_stream == 0;
+    }
   };
 
-  struct process {
+  constexpr generic_stream_ptr null_gsp {
-
+    .is_socket = false, .as_file_stream = 0
-    void end_process(unsigned exit_code);
-    void cleanup();
-
-    utility::list<thread *> threads;
-    utility::vector<string_pair> environment;
-    utility::id_allocator<stream *> open_streams;
-    utility::id_allocator<socket_listener *> socket_listeners;
-
-    uint64_t *p4;
-    uint64_t *p3;
-    uint64_t *p2s[512];
-    uint64_t **p1s[512];
-
-    bool **p1es_to_free_on_exit[512];
-
-    uint64_t p4_paddr;
-
-    //set to 0 if none
-    uint64_t framebuffer_vaddr;
-
-    //only valid if there are no threads
-    int32_t exit_code;
-
-    process();
-
-    //vaddr and paddr must be aligned, and vaddr must be < 0x0080.0000.0000
-    void map_page(uint64_t vaddr, uint64_t paddr,
-      bool write, bool execute, bool free_pram_on_exit);
-
-    //where "owned" means in lower half and marked free on exit
-    bool is_page_owned(uint64_t vaddr);
-
-    //returns start of first page
-    uint64_t get_free_vaddr_pages(uint64_t count);
-
-    //in lower half
-    uint64_t count_mapped_vram_pages();
-
   };
 
-  extern utility::id_allocator<process *> *processes;
+  extern utility::id_allocator<process *> *all_processes;
   extern utility::queue<thread *> *paused_threads;
-  extern utility::queue<thread *> *threads_waiting_for_input;
   extern thread *running_thread;
-  extern utility::list<socket_listener *> *all_socket_listeners;
-
-  stream_result create_application(
-    const vfile::vfile &file, process *&process_out, thread *&thread_out);
 
   void init_applications();
+  uint64_t add_process(process *p);
 
-  //returns true when resumed, false right now.
+  //returns new listener on success, 0 on failure. id is copied.
-  //must be called from non-interruptable syscall context.
+  socket_listener *try_register_socket_listener(const utility::string &id);
-  extern "C" bool save_thread_state(cpu_state &into);
+  //returns previously registered listener on success, 0 on failure.
+  socket_listener *try_get_socket_listener(const utility::string &id);
+  //this deletes the passed object as well.
+  void remove_socket_listener(socket_listener *sl);
 
-  //must be called from non-interruptable context
+  //saves running thread state, and switches to new task. when the thread is
-  [[noreturn]] void resume_next();
+  //resumed, returns to caller.
+  extern "C" void yield();
+
+  extern "C" [[noreturn]] void resume_next_thread();
+
+  class process {
+
+    utility::list<thread *> threads;
+    utility::id_allocator<generic_stream_ptr> open_streams;
+    utility::id_allocator<socket_listener *> running_socket_listeners;
+
+    struct string_pair {
+      utility::string a;
+      utility::string b;
+    };
+
+    utility::list<string_pair> environment_variables;
+
+  public:
+    app_memory *memory;
+
+    //set in get_framebuffer syscall
+    uint64_t framebuffer_vaddr = 0;
+
+    //set by add_process
+    uint64_t id;
+
+    //this class takes ownership of memory
+    process(app_memory *memory);
+    ~process();
+
+    //arguments are utility::move'd
+    void add_environment_variable(
+      utility::string &&name, utility::string &&value);
+
+    void add_thread(thread *t);
+    void notify_thread_ended(thread *t, int exit_code);
+    void on_end_process(int exit_code);
+    bool has_ended() const;
+
+    //these return handles
+    unsigned add_file_stream(file_stream *stream);
+    unsigned add_socket_stream(socket_stream_end *sse);
+    unsigned add_socket_listener(socket_listener *sl);
+
+    //if doesn't exist, returns null_gsp.
+    generic_stream_ptr get_stream(unsigned handle);
+
+    //assumes exists. just removes it from here
+    //and returns it. does no other cleanup
+    generic_stream_ptr take_stream(unsigned handle);
+
+    //assumes handle is not in use
+    void add_stream_with_handle(unsigned handle, generic_stream_ptr ptr);
+
+    //returns 0 if does not exist.
+    socket_listener *get_socket_listener(unsigned handle);
+
+    //just removes it from the list in here and returns it.
+    //does no other cleanup. returns 0 if does not exist.
+    socket_listener *take_socket_listener(unsigned handle);
+
+    void close_stream(unsigned handle);
+
+    //only meaningful if has_ended() is true
+    int exit_code;
+
+  };
+
+  class thread {
+
+    uint64_t stack_top;
+
+    //these four are set to 0 / false if the thread is not waiting.
+    //at most one of these can be non-zero / true at any time.
+    socket_stream_end *waiting_for_socket_stream;
+    socket_listener *waiting_to_accept_from;
+    socket_listener *waiting_to_connect_to;
+    bool waiting_for_input;
+
+    //this is only meaningful if the thread has just awoken from waiting to
+    //accept a connection from or connect to a socket stream. if the attempt
+    //was successful, new_socket_stream is set to the id of our end. otherwise,
+    //it is empty.
+    utility::maybe<unsigned> new_socket_stream_id;
+
+  public:
+    process *owner;
+
+    //the cpu state is saved here when the thread is not running.
+    cpu_state saved_state;
+
+    thread(process *owner, uint64_t entry);
+    //also call owner->notify_thread_ended, unless this
+    //is being called from process::on_end_process
+    void on_end_thread();
+
+    //these set up the necessary variables here and in the other object, save
+    //the cpu state, and then call resume_next. when the process is awoken by
+    //the other object, these return to their caller.
+    void wait_for_file_stream(file_stream *the_file_stream);
+    void wait_for_socket_stream(socket_stream_end *the_socket_stream);
+    utility::maybe<unsigned> wait_to_accept_from(
+      socket_listener *the_socket_listener);
+    utility::maybe<unsigned> wait_to_connect_to(
+      socket_listener *the_socket_listener);
+    void wait_for_input();
+
+    //these set the relevant variables and then wake up the thread
+    void notify_no_socket_stream();
+    void notify_new_socket_stream(unsigned id);
+
+  };
+
+  struct file_stream {
+    vfile::vfile the_file;
+    uint64_t offset;
+  };
+
+  struct socket {
+    utility::queue<uint8_t> queue_1;
+    utility::queue<uint8_t> queue_2;
+  };
+
+  struct socket_stream_end {
+    socket *the_socket;
+    utility::queue<uint8_t> &read_queue;
+    utility::queue<uint8_t> &write_queue;
+    utility::queue<thread *> waiting_to_read;
+    bool is_other_side_open;
+    process *other_process;
+    socket_stream_end *other_end;
+  };
+
+  struct socket_listener {
+    utility::queue<thread *> waiting_to_accept;
+    utility::queue<thread *> waiting_to_connect;
+  };
 
 }

kernel/include/hilbert/kernel/input.hpp

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <hilbert/kernel/application.hpp>
 #include <hilbert/kernel/utility.hpp>
 
 namespace hilbert::kernel::input {
@@ -37,9 +38,10 @@ namespace hilbert::kernel::input {
   };
 
   extern utility::queue<input_packet> *input_queue;
+  extern utility::queue<application::thread *> *waiting_for_input;
 
-  //notify a process waiting for input
+  //wake up the first process that was waiting for input, if any
-  void got_input();
+  void notify_waiting();
 
   //must be post switch to kernel page tables and mounting of file systems
   void init_input();

kernel/include/hilbert/kernel/load-app.hpp

@@ -0,0 +1,17 @@
+#pragma once
+
+#include <hilbert/kernel/app-memory.hpp>
+#include <hilbert/kernel/vfile.hpp>
+
+namespace hilbert::kernel {
+
+  enum class load_app_result {
+    success,
+    not_app,
+    io_error
+  };
+
+  load_app_result load_app(
+    vfile::vfile &file, app_memory &into, uint64_t &entry_out);
+
+}

kernel/include/hilbert/kernel/paging.hpp

@@ -15,12 +15,22 @@ namespace hilbert::kernel::paging {
 
   uint64_t find_unmapped_vram_region(uint64_t page_count);
 
-  uint64_t encode_pte(uint64_t addr, bool user, bool write, bool execute);
+  static inline uint64_t encode_pte(
+    uint64_t addr, bool user, bool write, bool execute) {
+    return (addr & 0x0000ffffffffffff) | (execute ? 0 : (1ULL << 63))
+         | (user << 2) | (write << 1) | 1;
+  }
+
+  static inline uint64_t pte_to_paddr(uint64_t pte) {
+    return pte & 0x0000fffffffff000;
+  }
 
   void init_kernel_page_tables(uint64_t kernel_offset);
 
   uint64_t take_pram_page();
 
+  void free_pram_page(uint64_t paddr);
+
   void map_kernel_stacks();
 
   void map_kernel_page(
@@ -28,12 +38,25 @@ namespace hilbert::kernel::paging {
 
   void unmap_kernel_page(uint64_t vaddr);
 
+  static inline void unmap_kernel_page(void *vaddr) {
+    unmap_kernel_page((uint64_t)vaddr);
+  }
+
   //maps writable and not executable
   void *map_new_kernel_pages(uint64_t count);
 
   //maps writable and not executable
   void map_new_kernel_page(uint64_t &vaddr_out, uint64_t &paddr_out);
 
+  //writable, not executable
+  template <class t>
+  static inline void map_new_kernel_page(
+    t *&vaddr_out, uint64_t &paddr_out) {
+    uint64_t vaddr_as_int;
+    map_new_kernel_page(vaddr_as_int, paddr_out);
+    vaddr_out = (t *)vaddr_as_int;
+  }
+
   uint64_t get_used_vram_page_count();
   uint64_t get_free_pram_page_count();
 

kernel/include/hilbert/kernel/utility.hpp

@@ -148,6 +148,14 @@ namespace hilbert::kernel::utility {
       delete n;
     }
 
+    void remove_first_of(const value_t &value) {
+      for (node *n = first; n; n = n->next)
+        if (n->value == value) {
+          remove(n);
+          return;
+        }
+    }
+
   };
 
   template <class value_t>
@@ -279,6 +287,12 @@ namespace hilbert::kernel::utility {
       return add(new value_t(move(v)));
     }
 
+    //pionter becomes owned by id_allocator
+    void add_id(value_t *v, unsigned i) {
+      the_vector.verify_buffer_len(i + 1);
+      the_vector.buffer[i] = v;
+    }
+
     bool has_id(unsigned i) {
       return i < the_vector.count && the_vector.buffer[i] != 0;
     }
@@ -287,6 +301,12 @@ namespace hilbert::kernel::utility {
       return *the_vector.buffer[i];
     }
 
+    value_t *take(unsigned i) {
+      value_t *v = the_vector.buffer[i];
+      the_vector.buffer[i] = 0;
+      return v;
+    }
+
     void remove_id(unsigned i) {
       delete the_vector.buffer[i];
       the_vector.buffer[i] = 0;
@@ -347,6 +367,26 @@ namespace hilbert::kernel::utility {
       return buffer[(count - 1 + next_read_index) % buffer_len];
     }
 
+    void remove(const value_t &v) {
+      unsigned written = 0;
+      for (unsigned i = 0; i < count; ++i) {
+        unsigned read_idx = (next_read_index + i) % buffer_len;
+        unsigned write_idx = (next_read_index + written) % buffer_len;
+        if (buffer[read_idx] != v) {
+          if (read_idx != write_idx)
+            buffer[write_idx] = utility::move(buffer[read_idx]);
+          ++written;
+        }
+      }
+      count = written;
+    }
+
+  };
+
+  template <class t>
+  struct maybe {
+    bool has_value;
+    t value;
   };
 
 }

kernel/include/hilbert/kernel/vfile.hpp

@@ -57,7 +57,7 @@ namespace hilbert::kernel::vfile {
 
   //path must be absolute. follows symlinks on all but the last node
   //always, and on the last node when follow_final_symlink is true.
-  storage::fs_result lookup_path(
+  storage::fs_result look_up_path(
     const canon_path &path, vfile &out, bool follow_final_symlink);
 
 }

kernel/makefile

@@ -2,7 +2,7 @@ SOURCES = \
   storage/bd/memory.cpp storage/fs/tarfs.cpp application.asm application.cpp \
   framebuffer.cpp interrupts.asm interrupts.cpp allocator.cpp storage.cpp \
   syscall.cpp utility.cpp paging.asm paging.cpp entry.cpp input.cpp panic.cpp \
-  vfile.cpp serial.asm
+  vfile.cpp serial.asm app-memory.cpp load-app.cpp
 
 build/%.asm.o: source/%.asm
 	@mkdir -p $(@D)

kernel/source/app-memory.cpp

@@ -0,0 +1,189 @@
+#include <hilbert/kernel/app-memory.hpp>
+#include <hilbert/kernel/paging.hpp>
+#include <hilbert/kernel/panic.hpp>
+
+namespace hilbert::kernel {
+
+  app_memory::app_memory() {
+
+    uint64_t p3_paddr;
+    paging::map_new_kernel_page(p3, p3_paddr);
+    paging::map_new_kernel_page(p4, p4_paddr);
+
+    for (int i = 0; i < 512; ++i) {
+      p4[i] = 0;
+      p3[i] = 0;
+      p2s[i] = 0;
+      p1s[i] = 0;
+      pram_pages_to_free_on_exit[i] = 0;
+    }
+
+    p4[0] = paging::encode_pte(p3_paddr, true, true, true);
+    p4[511] = paging::kernel_p4e;
+
+  }
+
+  app_memory::~app_memory() {
+
+    //first we see if the p2s exist
+    for (int p3i = 0; p3i < 512; ++p3i)
+      if (p3[p3i]) {
+
+        //now we see if the p1s under this p2 exist
+        for (int p2i = 0; p2i < 512; ++p2i)
+          if (p2s[p3i][p2i]) {
+
+            //we see if the pages under this p1 need to be freed
+            for (int p1i = 0; p1i < 512; ++p1i)
+              if (pram_pages_to_free_on_exit[p3i][p2i][p1i])
+                paging::free_pram_page(
+                  paging::pte_to_paddr(p1s[p3i][p2i][p1i]));
+
+            //we free the p1 and the pram list
+            paging::free_pram_page(paging::pte_to_paddr(p2s[p3i][p2i]));
+            paging::unmap_kernel_page((uint64_t)p1s[p3i][p2i]);
+            delete[] pram_pages_to_free_on_exit[p3i][p2i];
+
+          }
+
+        //free the p2, the p1 list, and the pram list list
+        paging::free_pram_page(paging::pte_to_paddr(p3[p3i]));
+        paging::unmap_kernel_page((uint64_t)p2s[p3i]);
+        delete[] p1s[p3i];
+        delete[] pram_pages_to_free_on_exit[p3i];
+
+      }
+
+    //finally, we free the p3 and the p4
+    paging::free_pram_page(paging::pte_to_paddr(p4[0]));
+    paging::unmap_kernel_page((uint64_t)p3);
+    paging::free_pram_page(p4_paddr);
+    paging::unmap_kernel_page((uint64_t)p4);
+
+  }
+
+  void app_memory::map_page(uint64_t vaddr, uint64_t paddr,
+    bool write, bool execute, bool free_pram_on_exit) {
+
+    int p1i = (vaddr >> 12) & 511;
+    int p2i = (vaddr >> 21) & 511;
+    int p3i = (vaddr >> 30) & 511;
+
+    if (p2s[p3i] == 0) {
+      uint64_t new_p2_paddr;
+      paging::map_new_kernel_page(p2s[p3i], new_p2_paddr);
+      p1s[p3i] = new v_page_table[512];
+      pram_pages_to_free_on_exit[p3i] = new bool *[512];
+      for (int i = 0; i < 512; ++i) {
+        p2s[p3i][i] = 0;
+        p1s[p3i][i] = 0;
+        pram_pages_to_free_on_exit[p3i][i] = 0;
+      }
+      p3[p3i] = paging::encode_pte(new_p2_paddr, true, true, true);
+    }
+
+    if (p1s[p3i][p2i] == 0) {
+      uint64_t new_p1_paddr;
+      paging::map_new_kernel_page(p1s[p3i][p2i], new_p1_paddr);
+      pram_pages_to_free_on_exit[p3i][p2i] = new bool[512];
+      for (int i = 0; i < 512; ++i) {
+        p1s[p3i][p2i][i] = 0;
+        pram_pages_to_free_on_exit[p3i][p2i][i] = false;
+      }
+      p2s[p3i][p2i] = paging::encode_pte(new_p1_paddr, true, true, true);
+    }
+
+    p1s[p3i][p2i][p1i] = paging::encode_pte(paddr, true, write, execute);
+    pram_pages_to_free_on_exit[p3i][p2i][p1i] = free_pram_on_exit;
+
+  }
+
+  void app_memory::unmap_page(uint64_t vaddr) {
+    int p1i = (vaddr >> 12) & 511;
+    int p2i = (vaddr >> 21) & 511;
+    int p3i = (vaddr >> 30) & 511;
+    if (pram_pages_to_free_on_exit[p3i][p2i][p1i]) {
+      pram_pages_to_free_on_exit[p3i][p2i][p1i] = false;
+      paging::free_pram_page(paging::pte_to_paddr(p1s[p3i][p2i][p1i]));
+    }
+    p1s[p3i][p2i][p1i] = 0;
+  }
+
+  bool app_memory::valid_to_read(
+    uint64_t vaddr_start, uint64_t vaddr_end, bool and_write) const {
+    if (vaddr_start > vaddr_end || vaddr_end > 0x8000000000)
+      return false;
+    vaddr_start = (vaddr_start / 4096) * 4096;
+    vaddr_end = (((vaddr_end - 1) / 4096) + 1) * 4096;
+    for (uint64_t vaddr = vaddr_start; vaddr < vaddr_end; ++vaddr) {
+      int p1i = (vaddr >> 12) & 511;
+      int p2i = (vaddr >> 21) & 511;
+      int p3i = (vaddr >> 30) & 511;
+      if (!p1s[p3i] || !p1s[p3i][p2i] || !(and_write
+            ? (p1s[p3i][p2i][p1i] & 0x1) : p1s[p3i][p2i][p1i]))
+        return false;
+    }
+    return true;
+  }
+
+  uint64_t app_memory::get_free_vaddr_pages(uint64_t count) {
+    uint64_t vaddr = 0x1000;
+    uint64_t run = 0;
+    while (true) {
+      if (run == count)
+        return vaddr;
+      if (vaddr + (run + 1) * 4096 > 0x4000000000)
+        //TODO: handle out of virtual memory
+        panic(0x9af5e6);
+      if (valid_to_read(vaddr + run * 4096, vaddr + (run + 1) * 4096, false)) {
+        vaddr += (run + 1) * 4096;
+        run = 0;
+      }
+      else
+        ++run;
+    }
+  }
+
+  uint64_t app_memory::map_new_stack() {
+    for (uint64_t base_vaddr = 0x4000000000;
+         base_vaddr < 0x8000000000; base_vaddr += 0x1000000)
+      if (!valid_to_read(base_vaddr + 4096, base_vaddr + 8192, false)) {
+
+        for (uint64_t vaddr = base_vaddr + 4096;
+             vaddr < base_vaddr + 0x1000000; vaddr += 4096) {
+
+          uint8_t *kvaddr;
+          uint64_t paddr;
+          paging::map_new_kernel_page(kvaddr, paddr);
+          for (int i = 0; i < 4096; ++i)
+            kvaddr[i] = 0;
+          paging::unmap_kernel_page(kvaddr);
+          map_page(vaddr, paddr, true, false, true);
+
+        }
+        return base_vaddr + 0x1000000;
+
+      }
+    //TODO: handle out of stacks
+    panic(0x9af5e6);
+
+  }
+
+  void app_memory::unmap_stack(uint64_t top) {
+    for (uint64_t vaddr = top - 0xfff000; vaddr < top; vaddr += 4096)
+      unmap_page(vaddr);
+  }
+
+  uint64_t app_memory::count_mapped_vram_pages() const {
+    uint64_t count = 0;
+    for (int p3i = 0; p3i < 512; ++p3i)
+      if (p3[p3i])
+        for (int p2i = 0; p2i < 512; ++p2i)
+          if (p2s[p3i][p2i])
+            for (int p1i = 0; p1i < 512; ++p1i)
+              if (p1s[p3i][p2i][p1i])
+                ++count;
+    return count;
+  }
+
+}

kernel/source/application.asm

@@ -137,9 +137,12 @@ resume_thread:
 
 extern copy_syscall_stack
 ;rdi = bottom
+;returns: pointer to copy
 
-global save_thread_state
+extern resume_next_thread
-save_thread_state:
+
+global yield
+yield:
 ;rdi = pointer to cpu state structure
 
   ;only saving registers that need to be preserved by this function
@@ -163,9 +166,7 @@ save_thread_state:
   mov qword [rdi + 152], rax ;kernel_stack_copy
   mov byte [rdi + 160], 0x01 ;in_syscall
 
-  xor al, al
+  jmp resume_next_thread
-  ret
 
 .resume_to:
-  mov al, 0x01
   ret

kernel/source/application.cpp

@@ -1,550 +1,282 @@
 #include <hilbert/kernel/application.hpp>
-#include <hilbert/kernel/paging.hpp>
+#include <hilbert/kernel/utility.hpp>
-#include <hilbert/kernel/panic.hpp>
+#include <hilbert/kernel/input.hpp>
-
-//TODO - scheduling.
 
 namespace hilbert::kernel::application {
 
-  process::process() : framebuffer_vaddr(0) {
+  //returns pointer to copy
-
+  extern "C" void *copy_syscall_stack(const uint8_t *bottom) {
-    uint64_t p4_vaddr;
+    uint64_t len = 0xfffffffffffff000 - (uint64_t)bottom;
-    paging::map_new_kernel_page(p4_vaddr, p4_paddr);
+    uint8_t *buffer = new uint8_t[len];
-    p4 = (uint64_t *)p4_vaddr;
+    for (uint64_t i = 0; i < len; ++i)
-
+      buffer[i] = bottom[i];
-    uint64_t p3_paddr;
+    return buffer;
-    uint64_t p3_vaddr;
-    paging::map_new_kernel_page(p3_vaddr, p3_paddr);
-    p3 = (uint64_t *)p3_vaddr;
-
-    for (int i = 1; i < 511; ++i)
-      p4[i] = 0;
-    p4[0] = paging::encode_pte(p3_paddr, true, true, true);
-    p4[511] = paging::kernel_p4e;
-
-    for (int i = 0; i < 512; ++i) {
-      p3[i] = 0;
-      p2s[i] = 0;
-      p1s[i] = 0;
-      p1es_to_free_on_exit[i] = 0;
-    }
-
   }
 
-  void process::map_page(uint64_t vaddr, uint64_t paddr,
+  extern "C" void restore_syscall_stack(uint8_t *from, uint8_t *to) {
-    bool write, bool execute, bool free_pram_on_exit
+    uint64_t len = 0xfffffffffffff000 - (uint64_t)to;
-  ) {
+    for (uint64_t i = 0; i < len; ++i)
-
+      to[i] = from[i];
-    uint64_t i = ((vaddr / 4096) / 512) / 512;
+    delete[] from;
-    uint64_t j = ((vaddr / 4096) / 512) % 512;
-    uint64_t k = (vaddr / 4096) % 512;
-
-    if (p2s[i] == 0) {
-      uint64_t p2_paddr;
-      uint64_t p2_vaddr;
-      paging::map_new_kernel_page(p2_vaddr, p2_paddr);
-      p3[i] = paging::encode_pte(p2_paddr, true, true, true);
-      p2s[i] = (uint64_t *)p2_vaddr;
-      p1s[i] = new uint64_t *[512];
-      p1es_to_free_on_exit[i] = new bool *[512];
-      for (int u = 0; u < 512; ++u) {
-        p2s[i][u] = 0;
-        p1s[i][u] = 0;
-        p1es_to_free_on_exit[i][u] = 0;
-      }
-    }
-
-    if (p2s[i][j] == 0) {
-      uint64_t p1_paddr;
-      uint64_t p1_vaddr;
-      paging::map_new_kernel_page(p1_vaddr, p1_paddr);
-      p2s[i][j] = paging::encode_pte(p1_paddr, true, true, true);
-      p1s[i][j] = (uint64_t *)p1_vaddr;
-      p1es_to_free_on_exit[i][j] = new bool[512];
-      for (int u = 0; u < 512; ++u) {
-        p1s[i][j][u] = 0;
-        p1es_to_free_on_exit[i][j][u] = false;
-      }
-    }
-
-    p1s[i][j][k] = paging::encode_pte(paddr, true, write, execute);
-    p1es_to_free_on_exit[i][j][k] = free_pram_on_exit;
-
   }
 
-  bool process::is_page_owned(uint64_t vaddr) {
+  utility::id_allocator<process *> *all_processes;
-    uint64_t i = ((vaddr / 4096) / 512) / 512;
-    uint64_t j = ((vaddr / 4096) / 512) % 512;
-    uint64_t k = (vaddr / 4096) % 512;
-    return
-      i < 512 && p1s[i] != 0 && p1s[i][j] != 0 &&
-      p1s[i][j][k] != 0 && p1es_to_free_on_exit[i][j][k];
-  }
-
-  uint64_t process::get_free_vaddr_pages(uint64_t count) {
-    uint64_t start = 0x200000 / 4096;
-    uint64_t length = 0;
-    while (start + length <= 0x8000000000 / 4096) {
-      if (length == count)
-        return start * 4096;
-      int i = ((start + length) / 512) / 512;
-      int j = ((start + length) / 512) % 512;
-      int k = (start + length) % 512;
-      if (p1s[i] == 0 || p1s[i][j] == 0 || p1s[i][j][k] == 0)
-        ++length;
-      else {
-        start += length + 1;
-        length = 0;
-      }
-    }
-    //TODO: handle out of memory
-    return 0;
-  }
-
-  uint64_t process::count_mapped_vram_pages() {
-    uint64_t count = 0;
-    for (int i = 0; i < 512; ++i)
-      if (p1s[i] != 0)
-        for (int j = 0; j < 512; ++j)
-          if (p1s[i][j] != 0)
-            for (int k = 0; k < 512; ++k)
-              if (p1s[i][j][k] != 0)
-                ++count;
-    return count;
-  }
-
-  utility::id_allocator<process *> *processes;
   utility::queue<thread *> *paused_threads;
-  utility::queue<thread *> *threads_waiting_for_input;
   thread *running_thread;
-  utility::list<socket_listener *> *all_socket_listeners;
 
-  static uint8_t correct_magic[16] = {
+  struct socket_listener_registration {
-    0x7f, 0x45, 0x4c, 0x46, 0x02, 0x01, 0x01, 0x00,
+    socket_listener *listener;
-    0x02, 0x00, 0x3e, 0x00, 0x01, 0x00, 0x00, 0x00
+    utility::string id;
   };
 
-#define READ(a, b, c) \
+  utility::list<socket_listener_registration> *all_running_socket_listeners;
-  { \
-    storage::fs_result _result = file.read_file(a, b, c); \
-    if (_result != storage::fs_result::success) \
-      return stream_result::io_error; \
-  }
-
-#define TRY_MAR(expr) \
-  { \
-    storage::fs_result _result = expr; \
-    if (_result != storage::fs_result::success) { \
-      delete process_out; \
-      return stream_result::io_error; \
-    } \
-  }
-
-  struct load_info {
-    uint64_t foffset;
-    uint64_t fsize;
-    uint64_t vaddr;
-    uint64_t voffset;
-    uint64_t vpages;
-    bool writable;
-    bool executable;
-  };
-
-  storage::fs_result map_and_read(
-    const vfile::vfile &file, process *process, uint64_t vaddr, uint64_t faddr,
-    uint64_t len, bool writable, bool executable
-  ) {
-
-    uint64_t page_vaddr = vaddr & ~4095;
-    int at_start = vaddr & 4095;
-    int at_end = 4096 - len - at_start;
-
-    uint64_t page_paddr = paging::take_pram_page();
-    process->map_page(page_vaddr, page_paddr, writable, executable, true);
-    uint64_t page_kvaddr = paging::find_unmapped_vram_region(1);
-    paging::map_kernel_page(page_paddr, page_kvaddr, true, false);
-
-    storage::fs_result result = storage::fs_result::success;
-
-    if (at_start) {
-      uint8_t *blank = (uint8_t *)page_kvaddr;
-      for (int i = 0; i < at_start; ++i)
-        blank[i] = 0;
-    }
-
-    if (len != 0)
-      result = file.read_file(faddr, len, (void *)(page_kvaddr + at_start));
-
-    if (at_end) {
-      uint8_t *blank = (uint8_t *)(page_kvaddr + at_start + len);
-      for (int i = 0; i < at_end; ++i)
-        blank[i] = 0;
-    }
-
-    paging::unmap_kernel_page(page_kvaddr);
-    return result;
-
-  }
-
-  stream_result create_application(
-    const vfile::vfile &file, process *&process_out, thread *&thread_out
-  ) {
-
-    uint8_t magic[16];
-    if (file.dir_entry.type != storage::file_type::regular_file)
-      return stream_result::not_a_regular_file;
-    if (file.dir_entry.length < 64)
-      return stream_result::not_an_executable;
-    READ(0, 8, magic)
-    READ(16, 8, magic + 8)
-    for (int i = 0; i < 16; ++i)
-      if (magic[i] != correct_magic[i])
-        return stream_result::not_an_executable;
-
-    uint64_t entry_point;
-    uint64_t phead_start;
-    uint16_t phead_entry_size;
-    uint16_t phead_entry_count;
-
-    READ(24, 8, &entry_point)
-    READ(32, 8, &phead_start)
-    READ(54, 2, &phead_entry_size)
-    READ(56, 2, &phead_entry_count)
-
-    if (file.dir_entry.length <
-        phead_start + phead_entry_size * phead_entry_count)
-      return stream_result::not_an_executable;
-
-    utility::vector<load_info> load_infos;
-
-    for (uint16_t i = 0; i < phead_entry_count; ++i) {
-
-      uint64_t entry_start = phead_start + phead_entry_size * i;
-
-      uint32_t seg_type;
-      READ(entry_start, 4, &seg_type)
-      if (seg_type != 1)
-        continue;
-
-      uint64_t foffset;
-      uint64_t vaddr;
-      uint64_t voffset;
-      uint64_t fsize;
-      uint64_t vsize;
-      uint32_t flags;
-
-      READ(entry_start + 8, 8, &foffset)
-      READ(entry_start + 16, 8, &vaddr)
-      READ(entry_start + 32, 8, &fsize)
-      READ(entry_start + 40, 8, &vsize)
-      READ(entry_start + 4, 4, &flags)
-
-      voffset = vaddr % 4096;
-      vaddr -= voffset;
-
-      if (vsize == 0)
-        continue;
-
-      if (file.dir_entry.length < foffset + fsize)
-        return stream_result::not_an_executable;
-      if (fsize > vsize)
-        return stream_result::not_an_executable;
-
-      if (vaddr < 0x200000)
-        return stream_result::not_an_executable;
-
-      uint64_t vpages = (voffset + vsize - 1) / 4096 + 1;
-
-      if (vaddr + vpages * 4096 > 0x8000000000)
-        return stream_result::not_an_executable;
-
-      load_info info = {
-        .foffset = foffset,
-        .fsize = fsize,
-        .vaddr = vaddr,
-        .voffset = voffset,
-        .vpages = vpages,
-        .writable = (flags & 2) == 2,
-        .executable = (flags & 1) == 1
-      };
-      load_infos.add_end(info);
-
-    }
-
-    process_out = new process();
-
-    for (unsigned i = 0; i < load_infos.count; ++i) {
-      const auto &info = load_infos.buffer[i];
-
-      uint64_t vaddr = info.vaddr + info.voffset;
-      uint64_t faddr = info.foffset;
-      uint64_t v_remaining = info.vpages * 4096 - info.voffset;
-      uint64_t f_remaining = info.fsize;
-
-      if (info.voffset != 0) {
-        int to_read = info.fsize < 4096 - info.voffset
-                    ? info.fsize : 4096 - info.voffset;
-        if (to_read > 0) {
-          TRY_MAR(map_and_read(file, process_out, vaddr, faddr, to_read,
-            info.writable, info.executable))
-          vaddr += to_read;
-          faddr += to_read;
-          v_remaining -= to_read;
-          f_remaining -= to_read;
-        }
-      }
-
-      while (f_remaining > 0) {
-        int to_read = f_remaining < 4096 ? f_remaining : 4096;
-        TRY_MAR(map_and_read(file, process_out, vaddr, faddr, to_read,
-          info.writable, info.executable))
-        vaddr += to_read;
-        faddr += to_read;
-        v_remaining -= to_read;
-        f_remaining -= to_read;
-      }
-
-      if (vaddr & 4095) {
-        v_remaining -= 4096 - (vaddr & 4095);
-        vaddr += 4096 - (vaddr & 4095);
-      }
-
-      while (v_remaining > 0) {
-        map_and_read(
-          file, process_out, vaddr, 0, 0, info.writable, info.executable);
-        vaddr += 4096;
-        v_remaining -= 4096;
-      }
-
-    }
-
-    for (uint64_t vaddr = 0x1000; vaddr < 0x1ff000; vaddr += 4096) {
-      uint64_t paddr = paging::take_pram_page();
-      uint64_t kvaddr = paging::find_unmapped_vram_region(1);
-      paging::map_kernel_page(paddr, kvaddr, true, false);
-      uint8_t *p = (uint8_t *)kvaddr;
-      for (int i = 0; i < 4096; ++i)
-        p[i] = 0;
-      paging::unmap_kernel_page(kvaddr);
-      process_out->map_page(vaddr, paddr, true, false, true);
-    }
-
-    thread_out = new thread();
-    process_out->threads.insert_end(thread_out);
-    thread_out->the_process = process_out;
-
-    thread_out->state = thread_state::paused;
-
-    thread_out->cpu.rax = 0;
-    thread_out->cpu.rbx = 0;
-    thread_out->cpu.rcx = 0;
-    thread_out->cpu.rdx = 0;
-    thread_out->cpu.rdi = 0;
-    thread_out->cpu.rsi = 0;
-    thread_out->cpu.rbp = 0;
-    thread_out->cpu.rsp = 0x1ff000;
-    thread_out->cpu.r8 = 0;
-    thread_out->cpu.r9 = 0;
-    thread_out->cpu.r10 = 0;
-    thread_out->cpu.r11 = 0;
-    thread_out->cpu.r12 = 0;
-    thread_out->cpu.r13 = 0;
-    thread_out->cpu.r14 = 0;
-    thread_out->cpu.r15 = 0;
-
-    thread_out->cpu.rflags = 0x200;
-    thread_out->cpu.rip = entry_point;
-    thread_out->cpu.cr3 = process_out->p4_paddr;
-    thread_out->cpu.in_syscall = false;
-
-    return stream_result::success;
-
-  }
 
   extern "C" void init_applications_asm();
 
   void init_applications() {
-    processes = new utility::id_allocator<process *>();
+    all_processes = new utility::id_allocator<process *>();
     paused_threads = new utility::queue<thread *>();
-    threads_waiting_for_input = new utility::queue<thread *>();
+    running_thread = 0;
-    all_socket_listeners = new utility::list<socket_listener *>();
+    all_running_socket_listeners =
+      new utility::list<socket_listener_registration>();
     init_applications_asm();
   }
 
-  //only called from non-interruptable contexts.
+  uint64_t add_process(process *p) {
+    return p->id = all_processes->add_new(utility::move(p));
+  }
+
+  socket_listener *try_register_socket_listener(const utility::string &id) {
+    for (auto *n = all_running_socket_listeners->first; n; n = n->next)
+      if (n->value.id == id)
+        return 0;
+    socket_listener *sl = new socket_listener();
+    all_running_socket_listeners->insert_end({.listener = sl, .id = id});
+    return sl;
+  }
+
+  socket_listener *try_get_socket_listener(const utility::string &id) {
+    for (auto *n = all_running_socket_listeners->first; n; n = n->next)
+      if (n->value.id == id)
+        return n->value.listener;
+    return 0;
+  }
+
+  void remove_socket_listener(socket_listener *sl) {
+    for (auto *n = all_running_socket_listeners->first; n; n = n->next)
+      if (n->value.listener == sl) {
+        all_running_socket_listeners->remove(n);
+        delete sl;
+        return;
+      }
+  }
+
   //cpu argument not on stack.
   extern "C" [[noreturn]] void resume_thread(const cpu_state &cpu);
 
-  extern "C" void *copy_syscall_stack(uint8_t *rsp) {
+  extern "C" [[noreturn]] void resume_next_thread() {
-    uint64_t size = 0xfffffffffffff000 - (uint64_t)rsp;
+    running_thread = 0;
-    uint8_t *buffer = new uint8_t[size];
-    for (uint64_t i = 0; i < size; ++i)
-      buffer[i] = rsp[i];
-    return buffer;
-  }
-
-  extern "C" void restore_syscall_stack(const uint8_t *from, uint8_t *rsp) {
-    uint64_t size = 0xfffffffffffff000 - (uint64_t)rsp;
-    for (uint64_t i = 0; i < size; ++i)
-      rsp[i] = from[i];
-    delete[] from;
-  }
-
-  thread::~thread() {
-    for (auto *p = the_process->threads.first; p; p = p->next)
-      if (p->value == this) {
-        the_process->threads.remove(p);
-        break;
-      }
-    if (the_process->threads.first == 0) {
-      the_process->exit_code = exit_code;
-      the_process->cleanup();
-    }
-    if (state != thread_state::running)
-      panic(0x9af5e6);
-  }
-
-  [[noreturn]] void resume_next() {
     while (paused_threads->count == 0)
       asm volatile ("sti\nhlt\ncli");
-    auto *t = paused_threads->take();
+    thread *t = paused_threads->take();
     running_thread = t;
-    t->state = thread_state::running;
+    resume_thread(t->saved_state);
-    resume_thread(t->cpu);
   }
 
-  void process::end_process(unsigned exit_code) {
+  process::process(app_memory *memory) : memory(memory) {}
-    while (threads.first != 0)
+
+  process::~process() {
+    delete memory; //:p
+  }
+
+  void process::add_environment_variable(
+    utility::string &&name, utility::string &&value) {
+    environment_variables.insert_end({.a = name, .b = value});
+  }
+
+  void process::add_thread(thread *t) {
+    threads.insert_end(t);
+  }
+
+  void process::notify_thread_ended(thread *t, int exit_code) {
+    threads.remove_first_of(t);
+    if (threads.first == 0)
+      on_end_process(exit_code);
+  }
+
+  void process::on_end_process(int exit_code) {
+
+    while (threads.first) {
+      threads.first->value->on_end_thread();
       delete threads.first->value;
+      threads.remove(threads.first);
+    }
+
+    //TODO: destroy file streams
+    //TODO: destroy socket streams
+    //TODO: destroy socket listeners
+
     this->exit_code = exit_code;
-    cleanup();
+
   }
 
-  void process::cleanup() {
+  bool process::has_ended() const {
-    //TODO
+    return threads.first == 0;
-    panic(0x9af5e6);
   }
 
-  socket_stream::socket_stream(socket *sock, bool are_we_b)
+  unsigned process::add_file_stream(file_stream *stream) {
-    : sock(sock), are_we_b(are_we_b),
+    return open_streams.add_new({
-      our_threads_waiting_to_read(are_we_b
+      .is_socket = false, .as_file_stream = stream });
-        ? sock->process_b_threads_waiting_to_read
-        : sock->process_a_threads_waiting_to_read),
-      their_threads_waiting_to_read(are_we_b
-        ? sock->process_a_threads_waiting_to_read
-        : sock->process_b_threads_waiting_to_read),
-      them_to_us(are_we_b ? sock->a_to_b : sock->b_to_a),
-      us_to_them(are_we_b ? sock->b_to_a : sock->a_to_b),
-      them_closed(are_we_b ? sock->a_closed : sock->b_closed),
-      us_closed(are_we_b ? sock->b_closed : sock->a_closed) {}
-
-  stream_result socket_stream::seek(seek_origin, int64_t) {
-    return stream_result::not_seekable;
   }
 
-  stream_result socket_stream::read(uint64_t count, void *into) {
+  unsigned process::add_socket_stream(socket_stream_end *stream) {
-    uint8_t *buffer = (uint8_t *)into;
+    return open_streams.add_new({
-    for (uint64_t i = 0; i < count; ++i) {
+      .is_socket = true, .as_socket_stream = stream });
-      while (them_to_us.count == 0) {
+  }
-        if (them_closed)
+
-          return stream_result::other_end_closed;
+  unsigned process::add_socket_listener(socket_listener *sl) {
-        if (!save_thread_state(running_thread->cpu)) {
+    return running_socket_listeners.add_new(utility::move(sl));
-          running_thread->state = thread_state::waiting;
+  }
-          our_threads_waiting_to_read.insert(running_thread);
+
-          resume_next();
+  generic_stream_ptr process::get_stream(unsigned handle) {
-        }
+    if (open_streams.has_id(handle))
+      return open_streams.get(handle);
+    return null_gsp;
+  }
+
+  generic_stream_ptr process::take_stream(unsigned handle) {
+    auto ptr = open_streams.get(handle);
+    open_streams.remove_id(handle);
+    return ptr;
+  }
+
+  void process::add_stream_with_handle(
+    unsigned handle, generic_stream_ptr ptr) {
+    open_streams.add_id(new generic_stream_ptr(ptr), handle);
+  }
+
+  socket_listener *process::get_socket_listener(unsigned handle) {
+    if (running_socket_listeners.has_id(handle))
+      return running_socket_listeners.get(handle);
+    return 0;
+  }
+
+  socket_listener *process::take_socket_listener(unsigned handle) {
+    if (running_socket_listeners.has_id(handle)) {
+      socket_listener *listener = running_socket_listeners.get(handle);
+      running_socket_listeners.remove_id(handle);
+      return listener;
+    }
+    return 0;
+  }
+
+  void process::close_stream(unsigned handle) {
+
+    if (!open_streams.has_id(handle))
+      return;
+    auto ptr = open_streams.get(handle);
+    open_streams.remove_id(handle);
+
+    if (ptr.is_socket) {
+      auto stream = ptr.as_socket_stream;
+
+      if (stream->is_other_side_open) {
+        stream->other_end->is_other_side_open = false;
+        auto &q = stream->other_end->waiting_to_read;
+        while (q.count > 0)
+          paused_threads->insert(q.take());
       }
-      buffer[i] = them_to_us.take();
+
+      else
+        delete stream->the_socket;
+
+      delete stream;
     }
-    return stream_result::success;
+
   }
 
-  stream_result socket_stream::write(uint64_t count, const void *from) {
+  thread::thread(process *owner, uint64_t entry)
-    if (them_closed)
+  : stack_top(owner->memory->map_new_stack()), waiting_for_socket_stream(0),
-      return stream_result::other_end_closed;
+    waiting_to_accept_from(0), waiting_to_connect_to(0),
-    const uint8_t *buffer = (const uint8_t *)from;
+    waiting_for_input(false), owner(owner) {
-    for (uint64_t i = 0; i < count; ++i) {
+
-      if (their_threads_waiting_to_read.count > 0) {
+    saved_state.rax = 0;
-        auto *ot = their_threads_waiting_to_read.take();
+    saved_state.rbx = 0;
-        ot->state = thread_state::paused;
+    saved_state.rcx = 0;
-        paused_threads->insert(ot);
+    saved_state.rdx = 0;
-      }
+    saved_state.rdi = 0;
-      us_to_them.insert(buffer[i]);
+    saved_state.rsi = 0;
-    }
+    saved_state.rbp = 0;
-    return stream_result::success;
+    saved_state.rsp = stack_top;
+    saved_state.r8 = 0;
+    saved_state.r9 = 0;
+    saved_state.r10 = 0;
+    saved_state.r11 = 0;
+    saved_state.r12 = 0;
+    saved_state.r13 = 0;
+    saved_state.r14 = 0;
+    saved_state.r15 = 0;
+
+    saved_state.rflags = 0x200;
+    saved_state.rip = entry;
+    saved_state.cr3 = owner->memory->p4_paddr;
+    saved_state.in_syscall = false;
+
   }
 
-  stream_result socket_stream::get_length(uint64_t &) {
+  void thread::on_end_thread() {
-    return stream_result::not_sized;
+    owner->memory->unmap_stack(stack_top);
+    if (waiting_for_socket_stream)
+      waiting_for_socket_stream->waiting_to_read.remove(this);
+    else if (waiting_to_accept_from)
+      waiting_to_accept_from->waiting_to_accept.remove(this);
+    else if (waiting_to_connect_to)
+      waiting_to_connect_to->waiting_to_connect.remove(this);
+    else if (waiting_for_input)
+      input::waiting_for_input->remove(this);
   }
 
-  stream_result socket_stream::set_length(uint64_t) {
+  void thread::wait_for_socket_stream(socket_stream_end *the_socket_stream) {
-    return stream_result::not_sized;
+    waiting_for_socket_stream = the_socket_stream;
+    the_socket_stream->waiting_to_read.insert(this);
+    yield();
+    waiting_for_socket_stream = 0;
   }
 
-  socket_stream::~socket_stream() {
+  utility::maybe<unsigned> thread::wait_to_accept_from(
-    if (our_threads_waiting_to_read.count > 0)
+    socket_listener *the_socket_listener) {
-      panic(0x9af5e6);
+    waiting_to_accept_from = the_socket_listener;
-    if (them_closed)
+    the_socket_listener->waiting_to_accept.insert(this);
-      delete sock;
+    yield();
-    else {
+    waiting_to_accept_from = 0;
-      us_closed = true;
+    return new_socket_stream_id;
-      while (their_threads_waiting_to_read.count > 0) {
-        auto *t = their_threads_waiting_to_read.take();
-        t->state = thread_state::paused;
-        paused_threads->insert(t);
-      }
-    }
   }
 
-  vfile_stream::vfile_stream(vfile::vfile &&file)
+  utility::maybe<unsigned> thread::wait_to_connect_to(
-    : file(utility::move(file)), offset(0) {}
+    socket_listener *the_socket_listener) {
-
+    waiting_to_connect_to = the_socket_listener;
-  stream_result vfile_stream::seek(seek_origin origin, int64_t offset) {
+    the_socket_listener->waiting_to_connect.insert(this);
-    uint64_t start_at = {};
+    yield();
-    switch (origin) {
+    waiting_to_connect_to = 0;
-    case seek_origin::beginning:
+    return new_socket_stream_id;
-      start_at = 0;
-      break;
-    case seek_origin::end:
-      start_at = file.dir_entry.length;
-      break;
-    case seek_origin::current_position:
-      start_at = this->offset;
-      break;
-    }
-    if (offset < 0 && (uint64_t)-offset > start_at)
-      return stream_result::out_of_bounds;
-    if (offset + start_at > file.dir_entry.length)
-      return stream_result::out_of_bounds;
-    this->offset = start_at + offset;
-    return stream_result::success;
   }
 
-  stream_result vfile_stream::read(uint64_t count, void *into) {
+  void thread::wait_for_input() {
-    if (offset + count > file.dir_entry.length)
+    waiting_for_input = false;
-      return stream_result::out_of_bounds;
+    input::waiting_for_input->insert(this);
-    if (file.read_file(offset, count, into) != storage::fs_result::success)
+    yield();
-      return stream_result::io_error;
+    waiting_for_input = false;
-    offset += count;
-    return stream_result::success;
   }
 
-  stream_result vfile_stream::write(uint64_t count, const void *from) {
+  void thread::notify_no_socket_stream() {
-    if (offset + count > file.dir_entry.length)
+    new_socket_stream_id.has_value = false;
-      return stream_result::out_of_bounds;
+    paused_threads->insert(this);
-    (void)from;
-    panic(0x9af5e6);
   }
 
-  stream_result vfile_stream::get_length(uint64_t &out) {
+  void thread::notify_new_socket_stream(unsigned id) {
-    out = file.dir_entry.length;
+    new_socket_stream_id.has_value = true;
-    return stream_result::success;
+    new_socket_stream_id.value = id;
-  }
+    paused_threads->insert(this);
-
-  stream_result vfile_stream::set_length(uint64_t to) {
-    (void)to;
-    panic(0x9af5e6);
   }
 
 }

kernel/source/entry.cpp

@@ -2,6 +2,7 @@
 #include <hilbert/kernel/storage/fs/tarfs.hpp>
 #include <hilbert/kernel/application.hpp>
 #include <hilbert/kernel/framebuffer.hpp>
+#include <hilbert/kernel/load-app.hpp>
 #include <hilbert/kernel/paging.hpp>
 #include <hilbert/kernel/serial.hpp>
 #include <hilbert/kernel/input.hpp>
@@ -201,25 +202,29 @@ extern "C" [[noreturn]] void entry() {
   vfile::canonize_path(init_path_string, init_path);
 
   vfile::vfile init_file;
-  if (vfile::lookup_path(init_path, init_file, true) !=
+  if (vfile::look_up_path(init_path, init_file, true) !=
       storage::fs_result::success)
     panic(0x7e874d);
 
-  application::process *init_process;
+  app_memory *init_memory = new app_memory();
-  application::thread *init_thread;
+  uint64_t init_entry_point;
-  if (application::create_application(init_file, init_process, init_thread) !=
+  load_app_result load_init_result =
-      application::stream_result::success)
+    load_app(init_file, *init_memory, init_entry_point);
+
+  if (load_init_result != load_app_result::success)
     panic(0xc39db3);
 
-  init_process->environment.add_end({
+  application::process *init_process = new application::process(init_memory);
-    .a = utility::string("ARGC", 4),
+  init_process->add_environment_variable(
-    .b = utility::string("1", 1)});
+    utility::string("ARGC", 4), utility::string("1", 1));
-  init_process->environment.add_end({
+  init_process->add_environment_variable(
-    .a = utility::string("ARGV0", 5),
+    utility::string("ARGV0", 5), utility::string("/bin/init", 9));
-    .b = utility::string("/bin/init", 9)});
+  application::add_process(init_process);
 
-  init_thread->state = application::thread_state::paused;
+  application::thread *init_thread =
+    new application::thread(init_process, init_entry_point);
+  init_process->add_thread(init_thread);
   application::paused_threads->insert(init_thread);
-  application::resume_next();
+  application::resume_next_thread();
 
 }

kernel/source/input.cpp

@@ -6,17 +6,16 @@
 namespace hilbert::kernel::input {
 
   utility::queue<input_packet> *input_queue;
+  utility::queue<application::thread *> *waiting_for_input;
+
+  void notify_waiting() {
+    if (waiting_for_input->count > 0)
+      application::paused_threads->insert(waiting_for_input->take());
+  }
 
   void init_input() {
     input_queue = new utility::queue<input_packet>();
-  }
+    waiting_for_input = new utility::queue<application::thread *>();
-
-  void got_input() {
-    if (application::threads_waiting_for_input->count > 0) {
-      auto *t = application::threads_waiting_for_input->take();
-      t->state = application::thread_state::paused;
-      application::paused_threads->insert(t);
-    }
   }
 
 }

kernel/source/interrupts.cpp

@@ -93,7 +93,7 @@ static void got_key(uint32_t key) {
   input::input_queue->insert({
     .keyboard = current_flags | key, .is_mouse = false});
 
-  input::got_input();
+  input::notify_waiting();
 
   if (key == (input::BREAK | 0x77))
     current_flags ^= input::NUM_LOCK;
@@ -229,6 +229,6 @@ extern "C" void on_mouse_interrupt(uint8_t byte) {
   else
     input::input_queue->insert(packet);
 
-  input::got_input();
+  input::notify_waiting();
 
 }

kernel/source/load-app.cpp

@@ -0,0 +1,153 @@
+#include <hilbert/kernel/load-app.hpp>
+#include <hilbert/kernel/paging.hpp>
+
+namespace hilbert::kernel {
+
+  struct elf_header {
+    uint8_t fixed[24];
+    uint64_t entry_point;
+    uint64_t program_header_offset;
+    uint64_t section_header_offset;
+    uint32_t flags;
+    uint16_t elf_header_length;
+    uint16_t program_header_pitch;
+    uint16_t program_header_count;
+  };
+
+  struct program_header {
+    uint32_t type;
+    uint32_t flags;
+    uint64_t foffset;
+    uint64_t vaddr;
+    uint64_t paddr;
+    uint64_t flength;
+    uint64_t vlength;
+  };
+
+  struct load_info {
+    uint64_t vaddr;
+    uint64_t vpages_start;
+    uint64_t vpages_count;
+    uint64_t foffset;
+    uint64_t flength;
+    bool writable;
+    bool executable;
+  };
+
+  static uint8_t expected_fixed_header[24] = {
+    0x7f, 0x45, 0x4c, 0x46, 0x02, 0x01, 0x01, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x02, 0x00, 0x3e, 0x00, 0x01, 0x00, 0x00, 0x00
+  };
+
+  load_app_result load_app(
+    vfile::vfile &file, app_memory &into, uint64_t &entry_out) {
+
+    if (file.dir_entry.type != storage::file_type::regular_file)
+      return load_app_result::not_app;
+
+    if (file.dir_entry.length < sizeof(elf_header))
+      return load_app_result::not_app;
+
+    elf_header eh;
+    if (file.read_file(0, sizeof(elf_header), &eh)
+        != storage::fs_result::success)
+      return load_app_result::io_error;
+
+    for (int i = 0; i < 24; ++i)
+      if (eh.fixed[i] != expected_fixed_header[i])
+        return load_app_result::not_app;
+
+    if (eh.entry_point < 0x1000 || eh.entry_point >= 0x4000000000)
+      return load_app_result::not_app;
+
+    utility::vector<load_info> load_infos;
+
+    for (int i = 0; i < eh.program_header_count; ++i) {
+
+      uint64_t offset = eh.program_header_offset + eh.program_header_pitch * i;
+      if (offset + sizeof(program_header) > file.dir_entry.length)
+        return load_app_result::not_app;
+
+      program_header ph;
+      if (file.read_file(offset, sizeof(program_header), &ph)
+          != storage::fs_result::success)
+        return load_app_result::io_error;
+
+      if (ph.type == 1) {
+
+        uint64_t vpages_start = (ph.vaddr / 4096) * 4096;
+        uint64_t vpages_end = ((ph.vaddr + ph.vlength - 1) / 4096 + 1) * 4096;
+
+        if (vpages_start < 0x1000 || vpages_end >= 0x4000000000 ||
+            ph.foffset + ph.flength > file.dir_entry.length)
+          return load_app_result::not_app;
+
+        load_infos.add_end((load_info){
+          .vaddr = ph.vaddr,
+          .vpages_start = vpages_start,
+          .vpages_count = (vpages_end - vpages_start) / 4096,
+          .foffset = ph.foffset,
+          .flength = ph.flength,
+          .writable = (ph.flags & 2) != 0,
+          .executable = (ph.flags & 4) != 0 });
+
+      }
+
+    }
+
+    for (unsigned i = 0; i < load_infos.count; ++i) {
+      const auto &li = load_infos.buffer[i];
+
+      for (uint64_t pi = 0; pi < li.vpages_count; ++pi) {
+
+        uint64_t page_user_vaddr = li.vpages_start + pi * 4096;
+
+        uint64_t page_kernel_vaddr;
+        uint64_t page_paddr;
+        paging::map_new_kernel_page(page_kernel_vaddr, page_paddr);
+
+        uint8_t *ptr = (uint8_t *)page_kernel_vaddr;
+        int bytes_left = 4096;
+        int64_t foffset = page_user_vaddr - li.vaddr;
+
+        if (foffset < 0) {
+          int to_skip = -foffset;
+          for (int i = 0; i < to_skip; ++i)
+            ptr[i] = 0;
+          ptr += to_skip;
+          bytes_left -= to_skip;
+          foffset = 0;
+        }
+
+        int64_t left_in_file = li.flength - foffset;
+        if (left_in_file > 0) {
+          int to_read = left_in_file < bytes_left ? left_in_file : bytes_left;
+          if (file.read_file(li.foffset + foffset, to_read, ptr) !=
+              storage::fs_result::success) {
+            paging::unmap_kernel_page((uint64_t)page_kernel_vaddr);
+            paging::free_pram_page(page_paddr);
+            return load_app_result::io_error;
+          }
+          ptr += to_read;
+          bytes_left -= to_read;
+        }
+
+        if (bytes_left > 0)
+          for (int i = 0; i < bytes_left; ++i)
+            ptr[i] = 0;
+
+        paging::unmap_kernel_page((uint64_t)page_kernel_vaddr);
+        into.map_page(
+          page_user_vaddr, page_paddr, li.writable, li.executable, true);
+
+      }
+
+    }
+
+    entry_out = eh.entry_point;
+    return load_app_result::success;
+
+  }
+
+}

kernel/source/paging.cpp

@@ -38,11 +38,6 @@ namespace hilbert::kernel::paging {
 
   uint64_t kernel_p4e;
 
-  uint64_t encode_pte(uint64_t addr, bool user, bool write, bool execute) {
-    return (addr & 0x0000ffffffffffff) | (execute ? 0 : (1ULL << 63))
-         | (user << 2) | (write << 1) | 1;
-  }
-
   void init_kernel_page_tables(uint64_t kernel_offset) {
     __kernel_p4_paddr = (uint64_t)kernel_p4 - kernel_offset;
     for (int i = 0; i < 511; ++i)
@@ -87,6 +82,11 @@ namespace hilbert::kernel::paging {
     return 0;
   }
 
+  void free_pram_page(uint64_t paddr) {
+    uint64_t page_i = paddr / 4096;
+    pram_usage_bitmap[page_i / 64] &= ~(1ULL << (page_i % 64));
+  }
+
   void map_kernel_stacks() {
     for (uint64_t vaddr = syscall_stack_bottom;
          vaddr < syscall_stack_top; vaddr += 4096)

kernel/source/syscall.cpp

@@ -1,5 +1,6 @@
 #include <hilbert/kernel/application.hpp>
 #include <hilbert/kernel/framebuffer.hpp>
+#include <hilbert/kernel/load-app.hpp>
 #include <hilbert/kernel/paging.hpp>
 #include <hilbert/kernel/input.hpp>
 #include <hilbert/kernel/panic.hpp>
@@ -17,16 +18,6 @@ namespace hilbert::kernel::syscall {
     file_result_directory
   };
 
-  bool is_range_owned_by_application(uint64_t start, uint64_t end) {
-    auto *process = application::running_thread->the_process;
-    uint64_t pstart = (start / 4096) * 4096;
-    uint64_t pend = ((end - 1) / 4096 + 1) * 4096;
-    for (uint64_t p = pstart; p < pend; p += 4096)
-      if (!process->is_page_owned(p))
-        return false;
-    return true;
-  }
-
   void set_zero(uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx) {
     rax = 0;
     rdi = 0;
@@ -35,26 +26,26 @@ namespace hilbert::kernel::syscall {
   }
 
   void encode_color_syscall(
-    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx
+    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx) {
-  ) {
+
     rax = (uint64_t)framebuffer::encode_color(
       rdi & 0xff, (rdi >> 8) & 0xff, (rdi >> 16) & 0xff);
     rdi = 0;
     rsi = 0;
     rdx = 0;
+
   }
 
   void get_framebuffer_syscall(
-    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx
+    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx) {
-  ) {
 
-    auto *process = application::running_thread->the_process;
+    auto *process = application::running_thread->owner;
     if (process->framebuffer_vaddr == 0) {
       uint64_t pages_needed =
         (framebuffer::dword_pitch * framebuffer::height * 4 - 1) / 4096 + 1;
-      uint64_t vaddr = process->get_free_vaddr_pages(pages_needed);
+      uint64_t vaddr = process->memory->get_free_vaddr_pages(pages_needed);
       for (uint64_t i = 0; i < pages_needed; ++i)
-        process->map_page(
+        process->memory->map_page(
           vaddr + i * 4096, framebuffer::paddr + i * 4096, true, false, false);
       process->framebuffer_vaddr = vaddr;
     }
@@ -69,23 +60,26 @@ namespace hilbert::kernel::syscall {
   }
 
   void open_file_syscall(
-    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx
+    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx) {
-  ) {
 
-    if (!is_range_owned_by_application(rdi, rdi + rsi)) {
+    const char *path = (const char *)rdi;
-      set_zero(rax, rdi, rsi, rdx);
+    uint64_t path_length = rsi;
-      return;
+    bool allow_creation = rdx & 1;
-    }
+    bool only_allow_creation = rdx & 2;
-
-    utility::string path_string((const char *)rdi, rsi);
 
     set_zero(rax, rdi, rsi, rdx);
 
+    auto *process = application::running_thread->owner;
+    if (!process->memory->valid_to_read(path, path + path_length, false))
+      return;
+
+    utility::string path_string(path, path_length);
+
     vfile::canon_path cp;
     vfile::vfile file;
     vfile::canonize_path(path_string, cp);
 
-    switch (vfile::lookup_path(cp, file, true)) {
+    switch (vfile::look_up_path(cp, file, true)) {
 
     case storage::fs_result::device_error:
     case storage::fs_result::fs_corrupt:
@@ -95,7 +89,7 @@ namespace hilbert::kernel::syscall {
 
     case storage::fs_result::does_not_exist:
 
-      if (!(rdx & 1)) {
+      if (!allow_creation) {
         rax = (uint64_t)application::stream_result::does_not_exist;
         return;
       }
@@ -105,7 +99,7 @@ namespace hilbert::kernel::syscall {
 
     case storage::fs_result::success:
 
-      if (rdx & 2) {
+      if (only_allow_creation) {
         rax = (uint64_t)application::stream_result::already_exists;
         return;
       }
@@ -115,33 +109,33 @@ namespace hilbert::kernel::syscall {
         return;
       }
 
+      rdi = process->add_file_stream(new application::file_stream {
+        .the_file = utility::move(file), .offset = 0 });
       rax = (uint64_t)application::stream_result::success;
-      rdi = application::running_thread->the_process->open_streams.add_new(
-        new application::vfile_stream(utility::move(file)));
-
-      return;
 
     }
 
   }
 
   void end_this_thread_syscall(
-    uint64_t &, uint64_t &rdi, uint64_t &, uint64_t &
+    uint64_t &, uint64_t &rdi, uint64_t &, uint64_t &) {
-  ) {
+
-    application::running_thread->exit_code = (int)(uint32_t)rdi;
+    int exit_code = (int)(uint32_t)rdi;
-    delete application::running_thread;
+    auto *t = application::running_thread;
-    application::resume_next();
+    t->on_end_thread();
+    t->owner->notify_thread_ended(t, exit_code);
+    delete t;
+    application::resume_next_thread();
   }
 
   void get_new_pages_syscall(
-    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx
+    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx) {
-  ) {
 
     uint64_t count = rdi;
     set_zero(rax, rdi, rsi, rdx);
 
-    auto *p = application::running_thread->the_process;
+    auto *p = application::running_thread->owner;
-    uint64_t vaddr = p->get_free_vaddr_pages(count);
+    uint64_t vaddr = p->memory->get_free_vaddr_pages(count);
 
     for (uint64_t i = 0; i < count; ++i) {
       uint64_t kvaddr;
@@ -150,7 +144,7 @@ namespace hilbert::kernel::syscall {
       for (int i = 0; i < 4096; ++i)
         ((uint8_t *)kvaddr)[i] = 0;
       paging::unmap_kernel_page((uint64_t)kvaddr);
-      p->map_page(vaddr + i * 4096, paddr, true, false, true);
+      p->memory->map_page(vaddr + i * 4096, paddr, true, false, true);
     }
 
     rax = vaddr;
@@ -158,229 +152,213 @@ namespace hilbert::kernel::syscall {
   }
 
   void get_input_packet_syscall(
-    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx
+    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx) {
-  ) {
 
     set_zero(rax, rdi, rsi, rdx);
-    auto *t = application::running_thread;
 
-    do
+    while (input::input_queue->count == 0)
-      if (input::input_queue->count > 0) {
+      application::running_thread->wait_for_input();
-        input::input_packet packet = input::input_queue->take();
-        if (packet.is_mouse) {
-          rax = packet.mouse.buttons | 0x80;
-          rdi = (uint16_t)packet.mouse.x_change;
-          rsi = (uint16_t)packet.mouse.y_change;
-        }
-        else {
-          rax = 0;
-          rdi = packet.keyboard;
-        }
-        return;
-      }
-    while (application::save_thread_state(t->cpu));
 
-    t->state = application::thread_state::waiting;
+    input::input_packet packet = input::input_queue->take();
-    application::threads_waiting_for_input->insert(t);
+    if (packet.is_mouse) {
-    application::resume_next();
+      rax = packet.mouse.buttons | 0x80;
+      rdi = (uint16_t)packet.mouse.x_change;
+      rsi = (uint16_t)packet.mouse.y_change;
+    }
+    else {
+      rax = 0;
+      rdi = packet.keyboard;
+    }
+
+  }
+
+  void create_socket(
+    application::process *p1, application::process *p2,
+    application::socket_stream_end *&se1_out,
+    application::socket_stream_end *&se2_out) {
+
+    application::socket *s = new application::socket();
+    se1_out = new application::socket_stream_end {
+      .the_socket = s, .read_queue = s->queue_1, .write_queue = s->queue_2,
+      .waiting_to_read = utility::queue<application::thread *>(),
+      .is_other_side_open = true, .other_process = p2, .other_end = 0 };
+    se2_out = new application::socket_stream_end {
+      .the_socket = s, .read_queue = s->queue_2, .write_queue = s->queue_2,
+      .waiting_to_read = utility::queue<application::thread *>(),
+      .is_other_side_open = true, .other_process = p1, .other_end = se1_out };
+    se1_out->other_end = se2_out;
 
   }
 
   void create_private_socket_syscall(
-    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx
+    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx) {
-  ) {
+
-    auto *s = new application::socket;
-    auto *ss1 = new application::socket_stream(s, false);
-    auto *ss2 = new application::socket_stream(s, true);
     set_zero(rax, rdi, rsi, rdx);
-    auto *p = application::running_thread->the_process;
+
-    rax = (uint64_t)p->open_streams.add_new(ss1);
+    auto *p = application::running_thread->owner;
-    rdi = (uint64_t)p->open_streams.add_new(ss2);
+
+    application::socket_stream_end *se1, *se2;
+    create_socket(p, p, se1, se2);
+
+    rax = p->add_socket_stream(se1);
+    rdi = p->add_socket_stream(se2);
+
   }
 
   void create_socket_listener_syscall(
-    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx
+    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx) {
-  ) {
 
-    if (!is_range_owned_by_application(rdi, rdi + rsi)) {
+    auto *p = application::running_thread->owner;
-      set_zero(rax, rdi, rsi, rdx);
-      return;
-    }
 
-    utility::string id_string((const char *)rdi, rsi);
+    const char *id = (const char *)rdi;
+    uint64_t id_length = rsi;
     set_zero(rax, rdi, rsi, rdx);
 
-    for (auto *p = application::all_socket_listeners->first; p; p = p->next)
+    if (!p->memory->valid_to_read(id, id + id_length, false))
-      if (p->value->id == id_string) {
+      return;
-        rax = (uint64_t)application::stream_result::socket_id_already_used;
-        return;
-      }
 
-    auto *sl = new application::socket_listener();
+    auto *sl =
-    sl->id = utility::move(id_string);
+      application::try_register_socket_listener(
-    sl->is_listening = true;
+        utility::string(id, id_length));
-    rax = (uint64_t)application::stream_result::success;
+
-    rdi = (uint64_t)application::running_thread->the_process
+    if (sl) {
-      ->socket_listeners.add_new(utility::move(sl));
+      rdx = p->add_socket_listener(sl);
+      rax = (uint64_t)application::stream_result::success;
+    }
+
+    else
+      rax = (uint64_t)application::stream_result::socket_id_already_in_use;
 
   }
 
   void stop_socket_listener_syscall(
-    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx
+    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx) {
-  ) {
 
     unsigned handle = (unsigned)rdi;
     set_zero(rax, rdi, rsi, rdx);
-    auto *p = application::running_thread->the_process;
 
-    if (p->socket_listeners.has_id(handle)) {
+    auto *sl =
-      auto *sl = p->socket_listeners.get(handle);
+      application::running_thread->owner->take_socket_listener(handle);
-      p->socket_listeners.remove_id(handle);
+
-      if (sl->waiting_to_accept_connection.count > 0 ||
+    if (!sl)
-          sl->waiting_to_connect.count > 0) {
+      return;
-        sl->is_listening = false;
+
-        while (sl->waiting_to_accept_connection.count > 0) {
+    while (sl->waiting_to_accept.count > 0)
-          auto *t = sl->waiting_to_accept_connection.take();
+      sl->waiting_to_accept.take()->notify_no_socket_stream();
-          t->state = application::thread_state::paused;
+    while (sl->waiting_to_connect.count > 0)
-          application::paused_threads->insert(t);
+      sl->waiting_to_connect.take()->notify_no_socket_stream();
-        }
+
-        while (sl->waiting_to_connect.count > 0) {
+    application::remove_socket_listener(sl);
-          auto *t = sl->waiting_to_connect.take();
-          t->state = application::thread_state::paused;
-          application::paused_threads->insert(t);
-        }
-      }
-      else
-        delete sl;
-    }
 
   }
 
   void accept_socket_connection_syscall(
-    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx
+    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx) {
-  ) {
 
     unsigned handle = (unsigned)rdi;
     set_zero(rax, rdi, rsi, rdx);
-    auto *t = application::running_thread;
-    auto *p = t->the_process;
 
-    if (!p->socket_listeners.has_id(handle)) {
+    auto *t = application::running_thread;
-      rax = (uint64_t)application::stream_result::socket_id_not_in_use;
+    auto *sl = t->owner->get_socket_listener(handle);
+
+    if (!sl) {
+      rax = (uint64_t)application::stream_result::bad_handle;
       return;
     }
 
-    auto *sl = p->socket_listeners.get(handle);
-
     if (sl->waiting_to_connect.count > 0) {
+
       auto *ot = sl->waiting_to_connect.take();
-      auto *sock = new application::socket();
+
-      application::stream *s1 = new application::socket_stream(sock, false);
+      application::socket_stream_end *our_end, *their_end;
-      application::stream *s2 = new application::socket_stream(sock, true);
+      create_socket(t->owner, ot->owner, our_end, their_end);
-      unsigned handle = p->open_streams.add_new(utility::move(s1));
+
-      ot->just_connected_to = s2;
+      unsigned our_handle = t->owner->add_socket_stream(our_end);
-      ot->state = application::thread_state::paused;
+      unsigned their_handle = ot->owner->add_socket_stream(their_end);
-      application::paused_threads->insert(ot);
+
+      ot->notify_new_socket_stream(their_handle);
+
       rax = (uint64_t)application::stream_result::success;
-      rdi = handle;
+      rdi = our_handle;
-      return;
+
     }
 
-    if (application::save_thread_state(t->cpu)) {
+    else {
-      if (sl->is_listening) {
+
+      auto result = t->wait_to_accept_from(sl);
+      if (result.has_value) {
         rax = (uint64_t)application::stream_result::success;
-        rdi = p->open_streams.add_new(utility::move(t->just_accepted));
+        rdi = result.value;
       }
-      else {
+      else
-        if (sl->waiting_to_accept_connection.count == 0 &&
-            sl->waiting_to_connect.count == 0)
-          delete sl;
         rax = (uint64_t)application::stream_result::socket_listener_closed;
-      }
-      return;
-    }
 
-    t->state = application::thread_state::waiting;
+    }
-    sl->waiting_to_accept_connection.insert(t);
-    application::resume_next();
 
   }
 
   void connect_to_socket_syscall(
-    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx
+    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx) {
-  ) {
 
-    if (!is_range_owned_by_application(rdi, rdi + rsi)) {
+    const char *id = (const char *)rdi;
-      set_zero(rax, rdi, rsi, rdx);
+    uint64_t id_length = rsi;
+    set_zero(rax, rdi, rsi, rdx);
+
+    auto *t = application::running_thread;
+
+    if (!t->owner->memory->valid_to_read(id, id + id_length, false))
+      return;
+
+    utility::string id_string(id, id_length);
+    auto *sl = application::try_get_socket_listener(id_string);
+
+    if (!sl) {
+      rax = (uint64_t)application::stream_result::socket_id_not_in_use;
       return;
     }
 
-    utility::string id_string((const char *)rdi, rsi);
+    if (sl->waiting_to_accept.count > 0) {
-    set_zero(rax, rdi, rsi, rdx);
 
-    for (auto *i = application::all_socket_listeners->first; i; i = i->next)
+      auto *ot = sl->waiting_to_accept.take();
-      if (i->value->id == id_string) {
-        auto *sl = i->value;
-        auto *t = application::running_thread;
-        auto *p = t->the_process;
 
-        if (sl->waiting_to_accept_connection.count > 0) {
+      application::socket_stream_end *our_end, *their_end;
-          auto *ot = sl->waiting_to_accept_connection.take();
+      create_socket(t->owner, ot->owner, our_end, their_end);
-          auto *sock = new application::socket();
-          auto *s1 = new application::socket_stream(sock, false);
-          auto *s2 = new application::socket_stream(sock, true);
-          unsigned handle = p->open_streams.add_new(utility::move(s1));
-          ot->just_accepted = s2;
-          ot->state = application::thread_state::paused;
-          application::paused_threads->insert(ot);
-          rax = (uint64_t)application::stream_result::success;
-          rdi = handle;
-          return;
-        }
 
-        if (application::save_thread_state(t->cpu)) {
+      unsigned our_handle = t->owner->add_socket_stream(our_end);
-          if (sl->is_listening) {
+      unsigned their_handle = ot->owner->add_socket_stream(their_end);
-            rax = (uint64_t)application::stream_result::success;
-            rdi = p->open_streams.add_new(utility::move(t->just_connected_to));
-          }
-          else {
-            if (sl->waiting_to_accept_connection.count == 0 &&
-                sl->waiting_to_connect.count == 0)
-              delete sl;
-            rax = (uint64_t)application::stream_result::socket_id_not_in_use;
-          }
-          return;
-        }
 
-        t->state = application::thread_state::waiting;
+      ot->notify_new_socket_stream(their_handle);
-        sl->waiting_to_connect.insert(t);
-        application::resume_next();
 
+      rax = (uint64_t)application::stream_result::success;
+      rdi = our_handle;
+
+    }
+
+    else {
+
+      auto result = t->wait_to_connect_to(sl);
+      if (result.has_value) {
+        rax = (uint64_t)application::stream_result::success;
+        rdi = result.value;
       }
+      else
+        rax = (uint64_t)application::stream_result::socket_listener_closed;
 
-    rax = (uint64_t)application::stream_result::socket_id_not_in_use;
+    }
 
   }
 
   void close_stream_syscall(
-    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx
+    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx) {
-  ) {
 
     unsigned handle = (unsigned)rdi;
     set_zero(rax, rdi, rsi, rdx);
-    auto *p = application::running_thread->the_process;
+    application::running_thread->owner->close_stream(handle);
-
-    if (p->open_streams.has_id(handle)) {
-      application::stream *s = p->open_streams.get(handle);
-      p->open_streams.remove_id(handle);
-      delete s;
-    }
 
   }
 
   void seek_stream_syscall(
-    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx
+    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx) {
-  ) {
 
     unsigned handle = (unsigned)rdi;
     uint8_t origin = (uint8_t)rsi;
@@ -390,117 +368,335 @@ namespace hilbert::kernel::syscall {
     if (origin >= 3)
       return;
 
-    auto *p = application::running_thread->the_process;
+    auto stream = application::running_thread->owner->get_stream(handle);
 
-    if (!p->open_streams.has_id(handle)) {
+    if (stream.is_null()) {
       rax = (uint64_t)application::stream_result::bad_handle;
       return;
     }
 
-    rax = (uint64_t)p->open_streams.get(handle)
+    if (stream.is_socket) {
-      ->seek((application::seek_origin)origin, offset);
+      rax = (uint64_t)application::stream_result::not_seekable;
+      return;
+    }
+
+    auto *fs = stream.as_file_stream;
+
+    switch (origin) {
+
+    case 0://beginning
+      if (offset < 0 || (uint64_t)offset > fs->the_file.dir_entry.length) {
+        rax = (uint64_t)application::stream_result::out_of_bounds;
+        return;
+      }
+      fs->offset = offset;
+      rax = (uint64_t)application::stream_result::success;
+      return;
+
+    case 1://end
+      if (offset > 0 || (uint64_t)-offset > fs->the_file.dir_entry.length) {
+        rax = (uint64_t)application::stream_result::out_of_bounds;
+        return;
+      }
+      fs->offset = fs->the_file.dir_entry.length + offset;
+      rax = (uint64_t)application::stream_result::success;
+      return;
+
+    case 2://current position
+      int64_t new_offset = offset + fs->offset;
+      if (new_offset < 0 || (uint64_t)new_offset > fs->the_file.dir_entry.length) {
+        rax = (uint64_t)application::stream_result::out_of_bounds;
+        return;
+      }
+      fs->offset = (uint64_t)new_offset;
+      rax = (uint64_t)application::stream_result::success;
+      return;
+
+    }
 
   }
 
   void read_from_stream_syscall(
-    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx
+    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx) {
-  ) {
 
     unsigned handle = (unsigned)rdi;
     uint64_t count = (uint64_t)rsi;
-    uint64_t buffer = (uint64_t)rdx;
+    uint8_t *buffer = (uint8_t *)rdx;
     set_zero(rax, rdi, rsi, rdx);
 
-    if (!is_range_owned_by_application(buffer, buffer + count))
+    auto *t = application::running_thread;
+
+    if (!t->owner->memory->valid_to_read(buffer, buffer + count, true))
       return;
 
-    auto *p = application::running_thread->the_process;
+    auto stream = t->owner->get_stream(handle);
 
-    if (!p->open_streams.has_id(handle)) {
+    if (stream.is_null()) {
       rax = (uint64_t)application::stream_result::bad_handle;
       return;
     }
 
-    rax = (uint64_t)p->open_streams.get(handle)->read(count, (void *)buffer);
+    if (stream.is_socket) {
+      auto *ss = stream.as_socket_stream;
+      for (uint64_t i = 0; i < count; ++i) {
+        while (ss->read_queue.count == 0) {
+          if (!ss->is_other_side_open) {
+            rax = (uint64_t)application::stream_result::other_end_closed;
+            return;
+          }
+          t->wait_for_socket_stream(ss);
+        }
+        buffer[i] = ss->read_queue.take();
+      }
+      rax = (uint64_t)application::stream_result::success;
+    }
+
+    else {
+      auto *fs = stream.as_file_stream;
+      if (fs->offset + count > fs->the_file.dir_entry.length) {
+        rax = (uint64_t)application::stream_result::out_of_bounds;
+        return;
+      }
+      auto read_result = fs->the_file.read_file(fs->offset, count, buffer);
+      if (read_result != storage::fs_result::success) {
+        rax = (uint64_t)application::stream_result::io_error;
+        return;
+      }
+      fs->offset += count;
+      rax = (uint64_t)application::stream_result::success;
+    }
 
   }
 
   void write_to_stream_syscall(
-    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx
+    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx) {
-  ) {
 
     unsigned handle = (unsigned)rdi;
     uint64_t count = (uint64_t)rsi;
-    uint64_t buffer = (uint64_t)rdx;
+    const uint8_t *buffer = (uint8_t *)rdx;
     set_zero(rax, rdi, rsi, rdx);
 
-    if (!is_range_owned_by_application(buffer, buffer + count))
+    auto *t = application::running_thread;
+
+    if (!t->owner->memory->valid_to_read(buffer, buffer + count, false))
       return;
 
-    auto *p = application::running_thread->the_process;
+    auto stream = t->owner->get_stream(handle);
 
-    if (!p->open_streams.has_id(handle)) {
+    if (stream.is_null()) {
       rax = (uint64_t)application::stream_result::bad_handle;
       return;
     }
 
-    rax = (uint64_t)p->open_streams.get(handle)
+    if (stream.is_socket) {
-      ->write(count, (const void *)buffer);
+      auto *ss = stream.as_socket_stream;
+      if (!ss->is_other_side_open) {
+        rax = (uint64_t)application::stream_result::other_end_closed;
+        return;
+      }
+      for (uint64_t i = 0; i < count; ++i)
+        ss->write_queue.insert(buffer[i]);
+      rax = (uint64_t)application::stream_result::success;
+    }
+
+    else {
+      //TODO: write to file
+      panic(0x9af5e6);
+    }
 
   }
 
   void get_stream_length_syscall(
-    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx
+    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx) {
-  ) {
 
     unsigned handle = (unsigned)rdi;
     set_zero(rax, rdi, rsi, rdx);
 
-    auto *p = application::running_thread->the_process;
+    auto *t = application::running_thread;
 
-    if (!p->open_streams.has_id(handle)) {
+    auto stream = t->owner->get_stream(handle);
+
+    if (stream.is_null())
       rax = (uint64_t)application::stream_result::bad_handle;
-      return;
-    }
 
-    rax = (uint64_t)p->open_streams.get(handle)->get_length(rdi);
+    else if (stream.is_socket)
+      rax = (uint64_t)application::stream_result::not_sized;
+
+    else {
+      rdi = stream.as_file_stream->the_file.dir_entry.length;
+      rax = (uint64_t)application::stream_result::success;
+    }
 
   }
 
+  struct env_var_spec {
+    uint64_t name_len;
+    const char *name;
+    uint64_t value_len;
+    const char *value;
+  };
+
+  struct gstream_spec {
+    uint64_t parent_handle;
+    uint64_t child_handle;
+  };
+
+  struct psi_spec {
+    uint64_t path_len;
+    const char *path;
+    uint64_t env_var_count;
+    const env_var_spec *env_vars;
+    uint64_t gstream_count;
+    const gstream_spec *gstreams;
+  };
+
   void start_process_syscall(
-    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx
+    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx) {
-  ) {
+
-    //TODO
+    const psi_spec *psi = (const psi_spec *)rdi;
-    (void)rax;
+    set_zero(rax, rdi, rsi, rdx);
-    (void)rdi;
+
-    (void)rsi;
+    auto *owner = application::running_thread->owner;
-    (void)rdx;
+
-    panic(0x9af5e6);
+    if (!owner->memory->valid_to_read(psi, psi + 1, false) ||
+        !owner->memory->valid_to_read(
+          psi->path, psi->path + psi->path_len, false) ||
+        !owner->memory->valid_to_read(
+          psi->env_vars, psi->env_vars + psi->env_var_count, false) ||
+        !owner->memory->valid_to_read(
+          psi->gstreams, psi->gstreams + psi->gstream_count, false))
+      return;
+
+    for (uint64_t i = 0; i < psi->env_var_count; ++i)
+      if (!owner->memory->valid_to_read(psi->env_vars[i].name,
+            psi->env_vars[i].name + psi->env_vars[i].name_len, false) ||
+          !owner->memory->valid_to_read(psi->env_vars[i].value,
+            psi->env_vars[i].value + psi->env_vars[i].value_len, false))
+        return;
+
+    for (uint64_t i = 0; i < psi->gstream_count; ++i) {
+      auto owner_stream = owner->get_stream(psi->gstreams[i].parent_handle);
+      if (owner_stream.is_null() || (owner_stream.is_socket &&
+            owner_stream.as_socket_stream->waiting_to_read.count != 0) ||
+          psi->gstreams[i].child_handle >= 65536)
+        return;
+    }
+
+    utility::string path_string(psi->path, psi->path_len);
+    vfile::canon_path cpath;
+    vfile::canonize_path(path_string, cpath);
+
+    vfile::vfile file;
+    auto lookup_result = vfile::look_up_path(cpath, file, true);
+    switch (lookup_result) {
+    case storage::fs_result::does_not_exist:
+      rax = (uint64_t)application::stream_result::does_not_exist;
+      return;
+    case storage::fs_result::device_error:
+    case storage::fs_result::fs_corrupt:
+      rax = (uint64_t)application::stream_result::io_error;
+      return;
+    case storage::fs_result::success:
+      break;
+    }
+
+    app_memory *memory = new app_memory();
+    uint64_t entry_point;
+    load_app_result load_result = load_app(file, *memory, entry_point);
+
+    switch (load_result) {
+    case load_app_result::io_error:
+      rax = (uint64_t)application::stream_result::io_error;
+      delete memory;
+      return;
+    case load_app_result::not_app:
+      rax = (uint64_t)application::stream_result::not_an_executable;
+      delete memory;
+      return;
+    case load_app_result::success:
+      break;
+    }
+
+    application::process *p = new application::process(memory);
+
+    for (uint64_t i = 0; i < psi->env_var_count; ++i)
+      p->add_environment_variable(
+        utility::string(psi->env_vars[i].name, psi->env_vars[i].name_len),
+        utility::string(psi->env_vars[i].value, psi->env_vars[i].value_len));
+
+    for (uint64_t i = 0; i < psi->gstream_count; ++i) {
+      auto s = owner->take_stream(psi->gstreams[i].parent_handle);
+      if (s.is_socket && s.as_socket_stream->is_other_side_open)
+        s.as_socket_stream->other_end->other_process = p;
+      p->add_stream_with_handle(psi->gstreams[i].child_handle, s);
+    }
+
+    rax = (uint64_t)application::stream_result::success;
+    rdi = application::add_process(p);
+    application::thread *t = new application::thread(p, entry_point);
+    p->add_thread(t);
+    application::paused_threads->insert(t);
+
   }
 
   void end_this_process_syscall(
-    uint64_t &, uint64_t &rdi, uint64_t &, uint64_t &
+    uint64_t &, uint64_t &rdi, uint64_t &, uint64_t &) {
-  ) {
+
-    application::running_thread->the_process->end_process((unsigned)rdi);
+    int exit_code = (int32_t)(uint32_t)rdi;
-    application::resume_next();
+    auto *p = application::running_thread->owner;
+    p->on_end_process(exit_code);
+    application::resume_next_thread();
+
   }
 
   void set_stream_length_syscall(
-    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx
+    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx) {
-  ) {
 
     unsigned handle = (unsigned)rdi;
-    uint64_t new_length = rsi;
     set_zero(rax, rdi, rsi, rdx);
 
-    auto *p = application::running_thread->the_process;
+    auto *t = application::running_thread;
 
-    if (!p->open_streams.has_id(handle)) {
+    auto stream = t->owner->get_stream(handle);
+
+    if (stream.is_null())
       rax = (uint64_t)application::stream_result::bad_handle;
-      return;
+
+    else if (stream.is_socket)
+      rax = (uint64_t)application::stream_result::not_sized;
+
+    else {
+      //TODO
+      panic(0x9af5e6);
     }
 
-    rax = (uint64_t)p->open_streams.get(handle)->set_length(new_length);
+  }
+
+  void get_other_end_process_handle_syscall(
+    uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx) {
+
+    unsigned handle = (unsigned)rdi;
+    set_zero(rax, rdi, rsi, rdx);
+
+    auto *t = application::running_thread;
+
+    auto stream = t->owner->get_stream(handle);
+
+    if (stream.is_null())
+      rax = (uint64_t)application::stream_result::bad_handle;
+
+    else if (!stream.is_socket)
+      rax = (uint64_t)application::stream_result::other_end_closed;
+
+    else {
+      auto s = stream.as_socket_stream;
+      if (!s->is_other_side_open)
+        rax = (uint64_t)application::stream_result::other_end_closed;
+      else {
+        rax = (uint64_t)application::stream_result::success;
+        rdi = (uint64_t)s->other_process->id;
+      }
+    }
 
   }
 
@@ -526,7 +722,8 @@ namespace hilbert::kernel::syscall {
     &get_stream_length_syscall,
     &start_process_syscall,
     &end_this_process_syscall,
-    &set_stream_length_syscall
+    &set_stream_length_syscall,
+    &get_other_end_process_handle_syscall
   };
 
   static constexpr int max_syscall_number = 19;
@@ -536,8 +733,7 @@ namespace hilbert::kernel::syscall {
 using namespace hilbert::kernel::syscall;
 
 extern "C" void do_syscall(
-  uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx
+  uint64_t &rax, uint64_t &rdi, uint64_t &rsi, uint64_t &rdx) {
-) {
 
   if (rax <= max_syscall_number && handlers[rax] != 0)
     handlers[rax](rax, rdi, rsi, rdx);

kernel/source/vfile.cpp

@@ -94,7 +94,7 @@ namespace hilbert::kernel::vfile {
     full_path.rel(target_path);
 
     vfile next;
-    RET_NOT_SUC(lookup_path(full_path, next, false))
+    RET_NOT_SUC(look_up_path(full_path, next, false))
 
     next.path = path;
     return next.follow_symlinks(out);
@@ -175,7 +175,7 @@ namespace hilbert::kernel::vfile {
     kernel::vfile::root = new vfile(root);
   }
 
-  storage::fs_result lookup_path(
+  storage::fs_result look_up_path(
     const canon_path &path, vfile &out, bool follow_final_symlink
   ) {
 

makefile

@@ -37,6 +37,7 @@ clean:
 	make -C euler clean
 	make -C kernel clean
 	make -C applications/init clean
+	make -C applications/goldman clean
 	make -C libraries/daguerre clean
 
 clean-dependencies: clean
@@ -57,7 +58,7 @@ ${MINTSUKI_HEADERS_DEP}:
 
 ${BINUTILS_DEP}:
 	mkdir -p dependencies toolchain/usr
-	test -e dependencies/binutils || (git clone --depth 1 -b binutils-2_42 https://sourceware.org/git/binutils-gdb dependencies/binutils && cd dependencies/binutils && git apply ../../patches/binutils.txt)
+	test -e dependencies/binutils || git clone --depth 1 -b binutils-2_42 https://sourceware.org/git/binutils-gdb dependencies/binutils
 	mkdir -p dependencies/binutils/build
 	cd dependencies/binutils/build && ../configure --disable-gdb \
 	  --target=x86_64-elf --prefix=${TOOLCHAIN_DIR}/usr
@@ -97,15 +98,19 @@ ${DAGUERRE_DEP}: ${LIBRARY_DEPS}
 kernel/build/kernel.elf: ${GCC_DEP} ${MINTSUKI_HEADERS_DEP} ${LIMINE_DEP}
 	+make -C kernel build/kernel.elf
 
-applications/init/build/init.elf: ${APP_DEPS} ${DAGUERRE_DEP}
+applications/init/build/init.elf: ${APP_DEPS}
 	+make -C applications/init build/init.elf
 
-build/initfs.tgz: applications/init/build/init.elf
+applications/goldman/build/goldman.elf: ${APP_DEPS}
+	+make -C applications/goldman build/goldman.elf
+
+build/initfs.tgz: applications/init/build/init.elf \
+                  applications/goldman/build/goldman.elf
 	@mkdir -p build
 	rm -rf build/initfs
 	cp -r skeleton build/initfs
-	mkdir build/initfs/bin
 	cp applications/init/build/init.elf build/initfs/bin/init
+	cp applications/goldman/build/goldman.elf build/initfs/bin/goldman
 	cd build/initfs && tar czf ../initfs.tgz .
 
 build/disk.iso: kernel/build/kernel.elf build/initfs.tgz ${LIMINE_DEP}

patches/binutils.txt

@@ -1,17 +0,0 @@
-diff --git a/ld/emulparams/elf_x86_64.sh b/ld/emulparams/elf_x86_64.sh
-index 466da2c4..806cd413 100644
---- a/ld/emulparams/elf_x86_64.sh
-+++ b/ld/emulparams/elf_x86_64.sh
-@@ -14,9 +14,9 @@ SCRIPT_NAME=elf
- ELFSIZE=64
- OUTPUT_FORMAT="elf64-x86-64"
- NO_REL_RELOCS=yes
--TEXT_START_ADDR=0x400000
--MAXPAGESIZE="CONSTANT (MAXPAGESIZE)"
--COMMONPAGESIZE="CONSTANT (COMMONPAGESIZE)"
-+TEXT_START_ADDR=0x200000
-+MAXPAGESIZE="0x1000"
-+COMMONPAGESIZE="0x1000"
- ARCH="i386:x86-64"
- MACHINE=
- TEMPLATE_NAME=elf

qemu.gdb

@@ -1,6 +1,5 @@
 target remote | qemu-system-x86_64 -gdb stdio -cdrom build/disk.iso -boot d
 symbol-file kernel/build/kernel.elf
-add-symbol-file build/initfs/bin/init
 set disassembly-flavor intel
 set print asm-demangle on
 layout src

readme.txt

@@ -15,7 +15,6 @@ acknowledgements (any under "dependencies" are downloaded during build):
       copyright 2024 free software foundation, inc.
       license: dependencies/binutils/COPYING (gnu gpl v2)
       homepage: https://www.gnu.org/software/binutils/
-      note: the patch in patches/binutils.txt is applied before building
 
   - dependencies/gcc (gnu compiler collection v14.1.0)
       copyright 2024 free software foundation, inc.
@@ -60,9 +59,12 @@ license in cc-by.txt (creative commons attribution 4.0 international):
 
 project structure:
 
+  - applications/goldman:
+      in the future, this will be the default compositor.
+
   - applications/init:
-      the initial program loaded by the kernel. currently it displays the image
+      the initial program loaded by the kernel. currently it just
-      by aaron burden, and inverts the colors when the enter key is pressed.
+      (attempts to) start /bin/compositor and then /bin/hello.
 
   - documentation:
       documentation. currently this directory is a bit disorganized, and has
@@ -78,8 +80,5 @@ project structure:
   - libraries/daguerre:
       an image loading / rendering library.
 
-  - patches/binutils.txt:
-      a patch that is applied to gnu binutils before it is built.
-
   - skeleton:
       files that are copied directly to the initfs.

skeleton/bin/compositor

@@ -0,0 +1 @@
+goldman

skeleton/cfg/init.sh

@@ -1 +0,0 @@
-/bin/wm &