rewrite application stuff in the kernel to support multitasking

7 files changed, 301 insertions, 144 deletions

diff --git a/kernel/include/hilbert/kernel/app-memory.hpp b/kernel/include/hilbert/kernel/app-memory.hpp
new file mode 100644
index 0000000..4f2f21d
--- /dev/null
+++ b/kernel/include/hilbert/kernel/app-memory.hpp
@@ -0,0 +1,61 @@
+#pragma once
+
+#include <stdint.h>
+
+//in the lower half, memory is only ever mapped below 0x0080.0000.0000. this is
+//one p3's worth. the p4 and p3 in app_memory are always real, but p3 may have
+//zero entries indicating p2's that aren't needed. similarly, the p2's may have
+//zero entries indicating p1's that aren't needed.
+
+namespace hilbert::kernel {
+
+  class app_memory {
+
+    typedef uint64_t *v_page_table;
+
+    v_page_table p4;
+    v_page_table p3;
+    v_page_table p2s[512];
+    v_page_table *p1s[512];
+
+    bool **pram_pages_to_free_on_exit[512];
+
+  public:
+    uint64_t p4_paddr;
+
+    app_memory();
+    ~app_memory();
+
+    //vaddr and paddr must be page aligned.
+    //vaddr must be < 0x0080.0000.0000.
+    void map_page(
+      uint64_t vaddr, uint64_t paddr, bool write,
+      bool execute, bool free_pram_on_exit);
+
+    //also frees the pram if marked in pram_pages_to_free_on_exit
+    void unmap_page(uint64_t vaddr);
+
+    bool valid_to_read(
+      uint64_t vaddr_start, uint64_t vaddr_end, bool and_write) const;
+
+    inline bool valid_to_read(
+      const void *vaddr_start, const void *vaddr_end, bool and_write) const {
+      return valid_to_read(
+        (uint64_t)vaddr_start, (uint64_t)vaddr_end, and_write);
+    }
+
+    //pages are together; returns start of first page.
+    //only looks in range [0x0000.0000.1000, 0x0040.0000.0000)
+    uint64_t get_free_vaddr_pages(uint64_t count);
+
+    //returns top of stack
+    uint64_t map_new_stack();
+    //also frees pram pages
+    void unmap_stack(uint64_t top);
+
+    //in lower half
+    uint64_t count_mapped_vram_pages() const;
+
+  };
+
+}
diff --git a/kernel/include/hilbert/kernel/application.hpp b/kernel/include/hilbert/kernel/application.hpp
index 7d48d8d..56c2a15 100644
--- a/kernel/include/hilbert/kernel/application.hpp
+++ b/kernel/include/hilbert/kernel/application.hpp
@@ -1,23 +1,14 @@
 #pragma once
 
+#include <hilbert/kernel/app-memory.hpp>
+#include <hilbert/kernel/utility.hpp>
 #include <hilbert/kernel/vfile.hpp>
 #include <stdint.h>
 
-//TODO: end application, threading.
-
 namespace hilbert::kernel::application {
 
-  class process;
-  class thread;
-
-  enum class thread_state {
-    running,
-    paused,
-    waiting
-  };
-
   enum class stream_result {
-    success,
+    success = 0,
     bad_handle,
     io_error,
     out_of_bounds,
@@ -26,7 +17,7 @@ namespace hilbert::kernel::application {
     not_an_executable,
     not_writable,
     not_seekable,
-    socket_id_already_used,
+    socket_id_already_in_use,
     socket_id_not_in_use,
     socket_listener_closed,
     other_end_closed,
@@ -34,83 +25,6 @@ namespace hilbert::kernel::application {
     not_sized
   };
 
-  enum class seek_origin {
-    beginning,
-    end,
-    current_position
-  };
-
-  class stream {
-  public:
-    virtual ~stream() {}
-    virtual stream_result seek(seek_origin origin, int64_t offset) = 0;
-    virtual stream_result read(uint64_t count, void *into) = 0;
-    virtual stream_result write(uint64_t count, const void *from) = 0;
-    virtual stream_result get_length(uint64_t &out) = 0;
-    virtual stream_result set_length(uint64_t to) = 0;
-  };
-
-  class vfile_stream : public stream {
-
-  private:
-    vfile::vfile file;
-    uint64_t offset;
-
-  public:
-    vfile_stream(vfile::vfile &&file);
-    virtual stream_result seek(seek_origin origin, int64_t offset) override;
-    virtual stream_result read(uint64_t count, void *into) override;
-    virtual stream_result write(uint64_t count, const void *from) override;
-    virtual stream_result get_length(uint64_t &out) override;
-    virtual stream_result set_length(uint64_t to) override;
-
-  };
-
-  struct socket {
-    utility::queue<thread *> process_a_threads_waiting_to_read;
-    utility::queue<thread *> process_b_threads_waiting_to_read;
-    utility::queue<uint8_t> a_to_b;
-    utility::queue<uint8_t> b_to_a;
-    bool a_closed;
-    bool b_closed;
-  };
-
-  class socket_stream : public stream {
-
-  private:
-    socket *sock;
-    bool are_we_b;
-
-    utility::queue<thread *> &our_threads_waiting_to_read;
-    utility::queue<thread *> &their_threads_waiting_to_read;
-    utility::queue<uint8_t> &them_to_us;
-    utility::queue<uint8_t> &us_to_them;
-    bool &them_closed;
-    bool &us_closed;
-
-  public:
-    socket_stream(socket *sock, bool are_we_b);
-    ~socket_stream();
-    virtual stream_result seek(seek_origin origin, int64_t offset) override;
-    virtual stream_result read(uint64_t count, void *into) override;
-    virtual stream_result write(uint64_t count, const void *from) override;
-    virtual stream_result get_length(uint64_t &out) override;
-    virtual stream_result set_length(uint64_t to) override;
-
-  };
-
-  struct string_pair {
-    utility::string a;
-    utility::string b;
-  };
-
-  struct socket_listener {
-    utility::string id;
-    utility::queue<thread *> waiting_to_accept_connection;
-    utility::queue<thread *> waiting_to_connect;
-    bool is_listening;
-  };
-
   struct [[gnu::packed]] cpu_state {
 
     uint64_t rax;
@@ -141,80 +55,180 @@ namespace hilbert::kernel::application {
 
   };
 
-  struct thread {
+  class process;
+  class thread;
+  class file_stream;
+  class socket;
+  class socket_stream_end;
+  class socket_listener;
+
+  struct generic_stream_ptr {
+    bool is_socket;
+    union {
+      file_stream *as_file_stream;
+      socket_stream_end *as_socket_stream;
+    };
+    bool is_null() {
+      return !is_socket && as_file_stream == 0;
+    }
+  };
+
+  constexpr generic_stream_ptr null_gsp {
+    .is_socket = false, .as_file_stream = 0
+  };
 
-    //propogated to process on destruction if this is the last thread.
-    int exit_code;
-    ~thread();
+  extern utility::id_allocator<process *> *all_processes;
+  extern utility::queue<thread *> *paused_threads;
+  extern thread *running_thread;
 
-    process *the_process;
-    thread_state state;
-    //only valid if paused or waiting
-    cpu_state cpu;
+  void init_applications();
+  uint64_t add_process(process *p);
 
-    stream *just_connected_to;
-    stream *just_accepted;
+  //returns new listener on success, 0 on failure. id is copied.
+  socket_listener *try_register_socket_listener(const utility::string &id);
+  //returns previously registered listener on success, 0 on failure.
+  socket_listener *try_get_socket_listener(const utility::string &id);
+  //this deletes the passed object as well.
+  void remove_socket_listener(socket_listener *sl);
 
-  };
+  //saves running thread state, and switches to new task. when the thread is
+  //resumed, returns to caller.
+  extern "C" void yield();
 
-  struct process {
+  extern "C" [[noreturn]] void resume_next_thread();
 
-    void end_process(unsigned exit_code);
-    void cleanup();
+  class process {
 
     utility::list<thread *> threads;
-    utility::vector<string_pair> environment;
-    utility::id_allocator<stream *> open_streams;
-    utility::id_allocator<socket_listener *> socket_listeners;
+    utility::id_allocator<generic_stream_ptr> open_streams;
+    utility::id_allocator<socket_listener *> running_socket_listeners;
 
-    uint64_t *p4;
-    uint64_t *p3;
-    uint64_t *p2s[512];
-    uint64_t **p1s[512];
+    struct string_pair {
+      utility::string a;
+      utility::string b;
+    };
 
-    bool **p1es_to_free_on_exit[512];
+    utility::list<string_pair> environment_variables;
 
-    uint64_t p4_paddr;
+  public:
+    app_memory *memory;
+
+    //set in get_framebuffer syscall
+    uint64_t framebuffer_vaddr = 0;
+
+    //set by add_process
+    uint64_t id;
+
+    //this class takes ownership of memory
+    process(app_memory *memory);
+    ~process();
+
+    //arguments are utility::move'd
+    void add_environment_variable(
+      utility::string &&name, utility::string &&value);
 
-    //set to 0 if none
-    uint64_t framebuffer_vaddr;
+    void add_thread(thread *t);
+    void notify_thread_ended(thread *t, int exit_code);
+    void on_end_process(int exit_code);
+    bool has_ended() const;
 
-    //only valid if there are no threads
-    int32_t exit_code;
+    //these return handles
+    unsigned add_file_stream(file_stream *stream);
+    unsigned add_socket_stream(socket_stream_end *sse);
+    unsigned add_socket_listener(socket_listener *sl);
 
-    process();
+    //if doesn't exist, returns null_gsp.
+    generic_stream_ptr get_stream(unsigned handle);
 
-    //vaddr and paddr must be aligned, and vaddr must be < 0x0080.0000.0000
-    void map_page(uint64_t vaddr, uint64_t paddr,
-      bool write, bool execute, bool free_pram_on_exit);
+    //assumes exists. just removes it from here
+    //and returns it. does no other cleanup
+    generic_stream_ptr take_stream(unsigned handle);
 
-    //where "owned" means in lower half and marked free on exit
-    bool is_page_owned(uint64_t vaddr);
+    //assumes handle is not in use
+    void add_stream_with_handle(unsigned handle, generic_stream_ptr ptr);
 
-    //returns start of first page
-    uint64_t get_free_vaddr_pages(uint64_t count);
+    //returns 0 if does not exist.
+    socket_listener *get_socket_listener(unsigned handle);
 
-    //in lower half
-    uint64_t count_mapped_vram_pages();
+    //just removes it from the list in here and returns it.
+    //does no other cleanup. returns 0 if does not exist.
+    socket_listener *take_socket_listener(unsigned handle);
+
+    void close_stream(unsigned handle);
+
+    //only meaningful if has_ended() is true
+    int exit_code;
 
   };
 
-  extern utility::id_allocator<process *> *processes;
-  extern utility::queue<thread *> *paused_threads;
-  extern utility::queue<thread *> *threads_waiting_for_input;
-  extern thread *running_thread;
-  extern utility::list<socket_listener *> *all_socket_listeners;
+  class thread {
 
-  stream_result create_application(
-    const vfile::vfile &file, process *&process_out, thread *&thread_out);
+    uint64_t stack_top;
 
-  void init_applications();
+    //these four are set to 0 / false if the thread is not waiting.
+    //at most one of these can be non-zero / true at any time.
+    socket_stream_end *waiting_for_socket_stream;
+    socket_listener *waiting_to_accept_from;
+    socket_listener *waiting_to_connect_to;
+    bool waiting_for_input;
+
+    //this is only meaningful if the thread has just awoken from waiting to
+    //accept a connection from or connect to a socket stream. if the attempt
+    //was successful, new_socket_stream is set to the id of our end. otherwise,
+    //it is empty.
+    utility::maybe<unsigned> new_socket_stream_id;
+
+  public:
+    process *owner;
+
+    //the cpu state is saved here when the thread is not running.
+    cpu_state saved_state;
+
+    thread(process *owner, uint64_t entry);
+    //also call owner->notify_thread_ended, unless this
+    //is being called from process::on_end_process
+    void on_end_thread();
+
+    //these set up the necessary variables here and in the other object, save
+    //the cpu state, and then call resume_next. when the process is awoken by
+    //the other object, these return to their caller.
+    void wait_for_file_stream(file_stream *the_file_stream);
+    void wait_for_socket_stream(socket_stream_end *the_socket_stream);
+    utility::maybe<unsigned> wait_to_accept_from(
+      socket_listener *the_socket_listener);
+    utility::maybe<unsigned> wait_to_connect_to(
+      socket_listener *the_socket_listener);
+    void wait_for_input();
+
+    //these set the relevant variables and then wake up the thread
+    void notify_no_socket_stream();
+    void notify_new_socket_stream(unsigned id);
 
-  //returns true when resumed, false right now.
-  //must be called from non-interruptable syscall context.
-  extern "C" bool save_thread_state(cpu_state &into);
+  };
+
+  struct file_stream {
+    vfile::vfile the_file;
+    uint64_t offset;
+  };
 
-  //must be called from non-interruptable context
-  [[noreturn]] void resume_next();
+  struct socket {
+    utility::queue<uint8_t> queue_1;
+    utility::queue<uint8_t> queue_2;
+  };
+
+  struct socket_stream_end {
+    socket *the_socket;
+    utility::queue<uint8_t> &read_queue;
+    utility::queue<uint8_t> &write_queue;
+    utility::queue<thread *> waiting_to_read;
+    bool is_other_side_open;
+    process *other_process;
+    socket_stream_end *other_end;
+  };
+
+  struct socket_listener {
+    utility::queue<thread *> waiting_to_accept;
+    utility::queue<thread *> waiting_to_connect;
+  };
 
 }
diff --git a/kernel/include/hilbert/kernel/input.hpp b/kernel/include/hilbert/kernel/input.hpp
index d1b7ca2..c92b947 100644
--- a/kernel/include/hilbert/kernel/input.hpp
+++ b/kernel/include/hilbert/kernel/input.hpp
@@ -1,5 +1,6 @@
 #pragma once
 
+#include <hilbert/kernel/application.hpp>
 #include <hilbert/kernel/utility.hpp>
 
 namespace hilbert::kernel::input {
@@ -37,9 +38,10 @@ namespace hilbert::kernel::input {
   };
 
   extern utility::queue<input_packet> *input_queue;
+  extern utility::queue<application::thread *> *waiting_for_input;
 
-  //notify a process waiting for input
-  void got_input();
+  //wake up the first process that was waiting for input, if any
+  void notify_waiting();
 
   //must be post switch to kernel page tables and mounting of file systems
   void init_input();
diff --git a/kernel/include/hilbert/kernel/load-app.hpp b/kernel/include/hilbert/kernel/load-app.hpp
new file mode 100644
index 0000000..1e41e31
--- /dev/null
+++ b/kernel/include/hilbert/kernel/load-app.hpp
@@ -0,0 +1,17 @@
+#pragma once
+
+#include <hilbert/kernel/app-memory.hpp>
+#include <hilbert/kernel/vfile.hpp>
+
+namespace hilbert::kernel {
+
+  enum class load_app_result {
+    success,
+    not_app,
+    io_error
+  };
+
+  load_app_result load_app(
+    vfile::vfile &file, app_memory &into, uint64_t &entry_out);
+
+}
diff --git a/kernel/include/hilbert/kernel/paging.hpp b/kernel/include/hilbert/kernel/paging.hpp
index 91ab5f7..f07913b 100644
--- a/kernel/include/hilbert/kernel/paging.hpp
+++ b/kernel/include/hilbert/kernel/paging.hpp
@@ -15,12 +15,22 @@ namespace hilbert::kernel::paging {
 
   uint64_t find_unmapped_vram_region(uint64_t page_count);
 
-  uint64_t encode_pte(uint64_t addr, bool user, bool write, bool execute);
+  static inline uint64_t encode_pte(
+    uint64_t addr, bool user, bool write, bool execute) {
+    return (addr & 0x0000ffffffffffff) | (execute ? 0 : (1ULL << 63))
+         | (user << 2) | (write << 1) | 1;
+  }
+
+  static inline uint64_t pte_to_paddr(uint64_t pte) {
+    return pte & 0x0000fffffffff000;
+  }
 
   void init_kernel_page_tables(uint64_t kernel_offset);
 
   uint64_t take_pram_page();
 
+  void free_pram_page(uint64_t paddr);
+
   void map_kernel_stacks();
 
   void map_kernel_page(
@@ -28,12 +38,25 @@ namespace hilbert::kernel::paging {
 
   void unmap_kernel_page(uint64_t vaddr);
 
+  static inline void unmap_kernel_page(void *vaddr) {
+    unmap_kernel_page((uint64_t)vaddr);
+  }
+
   //maps writable and not executable
   void *map_new_kernel_pages(uint64_t count);
 
   //maps writable and not executable
   void map_new_kernel_page(uint64_t &vaddr_out, uint64_t &paddr_out);
 
+  //writable, not executable
+  template <class t>
+  static inline void map_new_kernel_page(
+    t *&vaddr_out, uint64_t &paddr_out) {
+    uint64_t vaddr_as_int;
+    map_new_kernel_page(vaddr_as_int, paddr_out);
+    vaddr_out = (t *)vaddr_as_int;
+  }
+
   uint64_t get_used_vram_page_count();
   uint64_t get_free_pram_page_count();
 
diff --git a/kernel/include/hilbert/kernel/utility.hpp b/kernel/include/hilbert/kernel/utility.hpp
index c0b8c19..0247fee 100644
--- a/kernel/include/hilbert/kernel/utility.hpp
+++ b/kernel/include/hilbert/kernel/utility.hpp
@@ -148,6 +148,14 @@ namespace hilbert::kernel::utility {
       delete n;
     }
 
+    void remove_first_of(const value_t &value) {
+      for (node *n = first; n; n = n->next)
+        if (n->value == value) {
+          remove(n);
+          return;
+        }
+    }
+
   };
 
   template <class value_t>
@@ -279,6 +287,12 @@ namespace hilbert::kernel::utility {
       return add(new value_t(move(v)));
     }
 
+    //pionter becomes owned by id_allocator
+    void add_id(value_t *v, unsigned i) {
+      the_vector.verify_buffer_len(i + 1);
+      the_vector.buffer[i] = v;
+    }
+
     bool has_id(unsigned i) {
       return i < the_vector.count && the_vector.buffer[i] != 0;
     }
@@ -287,6 +301,12 @@ namespace hilbert::kernel::utility {
       return *the_vector.buffer[i];
     }
 
+    value_t *take(unsigned i) {
+      value_t *v = the_vector.buffer[i];
+      the_vector.buffer[i] = 0;
+      return v;
+    }
+
     void remove_id(unsigned i) {
       delete the_vector.buffer[i];
       the_vector.buffer[i] = 0;
@@ -347,6 +367,26 @@ namespace hilbert::kernel::utility {
       return buffer[(count - 1 + next_read_index) % buffer_len];
     }
 
+    void remove(const value_t &v) {
+      unsigned written = 0;
+      for (unsigned i = 0; i < count; ++i) {
+        unsigned read_idx = (next_read_index + i) % buffer_len;
+        unsigned write_idx = (next_read_index + written) % buffer_len;
+        if (buffer[read_idx] != v) {
+          if (read_idx != write_idx)
+            buffer[write_idx] = utility::move(buffer[read_idx]);
+          ++written;
+        }
+      }
+      count = written;
+    }
+
+  };
+
+  template <class t>
+  struct maybe {
+    bool has_value;
+    t value;
   };
 
 }
diff --git a/kernel/include/hilbert/kernel/vfile.hpp b/kernel/include/hilbert/kernel/vfile.hpp
index b64ae63..a092e39 100644
--- a/kernel/include/hilbert/kernel/vfile.hpp
+++ b/kernel/include/hilbert/kernel/vfile.hpp
@@ -57,7 +57,7 @@ namespace hilbert::kernel::vfile {
 
   //path must be absolute. follows symlinks on all but the last node
   //always, and on the last node when follow_final_symlink is true.
-  storage::fs_result lookup_path(
+  storage::fs_result look_up_path(
     const canon_path &path, vfile &out, bool follow_final_symlink);
 
 }