batman

backend/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "oracle"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+rocket = { version = "0.5.1", features = ["serde_json"] }
+ollama-rs = { version = "0.2.0", features = ["stream"] }
+allpaca-models = { workspace = true }
+reqwest = { workspace = true }
+tokio = { workspace = true }
+prost = { workspace = true }
+json = { workspace = true }
+lazy_static = "1.5.0"

backend/Dockerfile

@@ -0,0 +1,4 @@
+FROM rust:slim
+RUN apt update && apt install -y pkg-config librust-openssl-dev && apt clean
+RUN cargo install cargo-watch
+CMD [ "cargo", "watch", "-x", "run" ]

backend/Rocket.toml

@@ -0,0 +1,11 @@
+[default]
+address = "0.0.0.0"
+port = 80
+
+ip_header = "X-Real-IP"
+log_level = "normal"
+temp_dir = "/tmp"
+workers = 4
+
+[debug]
+log_level = "debug"

backend/src/main.rs

@@ -0,0 +1,56 @@
+#[macro_use] extern crate rocket;
+mod models;
+use models::list;
+
+use lazy_static::lazy_static;
+use tokio::sync::Mutex;
+use std::sync::Arc;
+use ollama_rs::Ollama;
+use rocket::{
+    http::{Header, Status},
+    response::Responder,
+    Request};
+
+lazy_static! {
+    pub static ref OLLAMA: Arc<Mutex<Ollama>> = Arc::new(Mutex::new(Ollama::new("http://ollama", 11434)));
+}
+pub static BASE: &'static str = "http://ollama:11434/api";
+
+#[macro_export]
+macro_rules! response {
+    ($name:ident, $body:ty) => {
+        pub struct $name {
+            pub status: rocket::http::Status,
+            pub body: $body,
+        }
+        #[rocket::async_trait]
+        impl<'r> Responder<'r, 'static> for $name {
+            fn respond_to(self, _request: &'r Request) -> rocket::response::Result<'static> {
+                let body: Vec<u8> = self.body.into();
+                rocket::response::Response::build()
+                    .header(Header::new("Content-Type", "application/x-protobuf"))
+                    .sized_body(body.len(), std::io::Cursor::new(body))
+                    .status(self.status)
+                    .ok()
+            }
+        }
+    };
+}
+
+response!(Response, &'static str);
+
+#[get("/")]
+async fn index() -> Response {
+    Response {
+        status: Status::ImATeapot,
+        body: "Hello, world!",
+    }
+}
+
+#[rocket::main]
+async fn main() -> Result<(), rocket::Error> {
+    rocket::build()
+        .mount("/", routes![index, models::list])
+        .launch().await?;
+    Ok(())
+}

backend/src/models.rs

@@ -0,0 +1,22 @@
+use rocket::{http::{Header, Status}, response::Responder, Request, Response};
+use json::JsonValue;
+
+use allpaca_models::{Models, Model};
+use super::{OLLAMA, BASE, response};
+
+response!(ListResponse, Models);
+
+#[get("/models/list")]
+pub async fn list() -> ListResponse {
+    let models = OLLAMA.lock().await.list_local_models().await.unwrap();
+    ListResponse {
+        status: Status::Ok,
+        body: Models(models.iter().map(|model| {
+            Model {
+                modified_at: model.modified_at.clone(),
+                size: model.size.to_string(),
+                name: model.name.clone(),
+            }
+        }).collect()),
+    }
+}