Fetchplz!

This commit is contained in:
soup 2024-10-18 00:29:46 -04:00
parent 523bbd2e4c
commit da94c17fe8
No known key found for this signature in database
12 changed files with 426 additions and 226 deletions

9
Cargo.lock generated
View file

@ -8,6 +8,15 @@ version = "1.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3"
[[package]]
name = "fetchplz"
version = "0.0.0"
dependencies = [
"http",
"httplz",
"httplzx",
]
[[package]]
name = "fnv"
version = "1.0.7"

View file

@ -7,9 +7,15 @@ description = "A sans-io, no-std HTTP implementation"
[workspace]
members = [
"crates/fetchplz",
"crates/httplzx"
]
[workspace.dependencies]
httplz = { path = ".", features = ["std"] }
httplzx = { path = "./crates/httplzx" }
fetchplz = { path = "./crates/fetchplz" }
[features]
std = []

View file

@ -0,0 +1,9 @@
[package]
name = "fetchplz"
version = "0.0.0"
edition = "2021"
[dependencies]
httplz = { workspace = true }
httplzx = { workspace = true }
http = { version = "1.1.0" }

181
crates/fetchplz/src/lib.rs Normal file
View file

@ -0,0 +1,181 @@
use std::io::{Error as IOError, Read, Write};
use std::net::TcpStream;
use std::str::FromStr;
use httplz::NeedsMoreData;
use httplzx::ToEvents;
pub struct Body {
buf: httplz::Buf<Box<[u8]>, u8>,
chunk_end: usize,
read_amt: usize,
stream: TcpStream,
conn: httplz::Connection,
}
impl Body {
fn find_next_chunk_sz(&mut self) -> Result<(), IOError> {
self.buf.pop_front(self.chunk_end);
self.chunk_end = 0;
self.read_amt = 0;
while self.conn.is_recving() {
dbg!(self.conn, String::from_utf8_lossy(self.buf.filled()));
let res = self.conn.poll_recv(self.buf.filled());
if res.needs_more_data() {
self.buf.read_from(&mut self.stream)?;
continue;
}
let (amt, ev) = res.map_err(|e| IOError::other(format!("{e:?}")))?;
match ev {
httplz::Event::BodyChunk(b) => {
self.chunk_end = b.len();
return Ok(());
},
_ => self.buf.pop_front(amt),
};
}
Ok(())
}
}
impl Read for Body {
fn read(&mut self, mut buf: &mut [u8]) -> std::io::Result<usize> {
if self.chunk_end == self.read_amt {
self.find_next_chunk_sz()?;
}
let chunk = &self.buf.filled()[self.read_amt..self.chunk_end];
let amt = buf.write(chunk)?;
self.read_amt += amt;
Ok(amt)
}
}
impl Body {
pub fn collect_bytes(self) -> Result<Vec<u8>, IOError> {
self.bytes().collect()
}
}
pub type Response = http::Response<Body>;
#[derive(Debug)]
pub enum Error {
IOError(IOError),
HttplzError(httplz::Error),
InvalidUri,
InvalidStatusCode,
InvalidHeaderValue,
}
impl From<IOError> for Error {
fn from(value: IOError) -> Self {
Self::IOError(value)
}
}
impl From<httplz::Error> for Error {
fn from(value: httplz::Error) -> Self {
Self::HttplzError(value)
}
}
pub fn fetch(path: &str) -> Result<Response, Error> {
let uri = http::Uri::from_str(path).map_err(|_| Error::InvalidUri)?;
let scheme = uri.scheme().cloned().unwrap_or(http::uri::Scheme::HTTPS);
let host = uri.host().ok_or(Error::InvalidUri)?;
let mut stream = TcpStream::connect(format!(
"{}:{}",
host,
uri.port_u16().unwrap_or_else(|| match scheme.as_str() {
"http" => 80,
_ => 443,
})
))?;
let mut conn = httplz::Connection::new(httplz::Role::Client);
let (mut parts, _) = http::Request::new(()).into_parts();
parts
.headers
.insert("Host", http::HeaderValue::from_str(host).unwrap());
parts.uri = uri;
let mut events = parts.to_events();
events.extend_from_slice(&[httplz::Event::HeadersDone, httplz::Event::SendDone]);
dbg!("a");
let mut buf = httplz::Buf::new(vec![0; 4096].into_boxed_slice());
for event in events {
conn.handle_send(&event, &mut buf)?;
}
stream.write_all(buf.filled())?;
assert!(conn.is_recving());
buf.clear();
let (mut parts, _) = http::Response::new(()).into_parts();
while conn.is_recving() {
let data = buf.filled();
let res = conn.poll_recv(data);
if res.needs_more_data() {
buf.read_from(&mut stream)?;
continue;
}
let (amt, event) = res?;
let mut brk = false;
match event {
httplz::Event::StatusLine(sl) => {
parts.status = http::StatusCode::from_u16(sl.status_code)
.map_err(|_| Error::InvalidStatusCode)?;
parts.version = http::Version::HTTP_11;
},
httplz::Event::Header(h) => {
parts.headers.insert(
http::HeaderName::from_str(h.name).unwrap(),
http::HeaderValue::from_bytes(h.value)
.map_err(|_| Error::InvalidHeaderValue)?,
);
},
httplz::Event::HeadersDone => brk = true,
_ => (),
};
dbg!(amt, brk);
buf.pop_front(amt);
if brk {
break;
}
}
Ok(http::Response::from_parts(
parts,
Body {
buf,
stream,
conn,
chunk_end: 0,
read_amt: 0,
},
))
}
#[cfg(test)]
mod test_fetch {
use crate::fetch;
#[test]
fn test_fetch_1() {
let resp = fetch("http://httpbin.org/get")
.unwrap()
.map(|b| b.collect_bytes().unwrap());
let body = resp.into_body();
let body = String::from_utf8_lossy(&body);
assert_eq!(
body,
include_str!("../testing/snapshots/httpbin_get_empty.txt")
)
}
}

View file

@ -0,0 +1,11 @@
{
"args": {},
"headers": {
"Accept": "*/*",
"Host": "httpbin.org",
"User-Agent": "curl/8.7.1",
"X-Amzn-Trace-Id": "Root=1-6711c17e-1fdd277564daedff276ec60a"
},
"origin": "72.94.94.124",
"url": "https://httpbin.org/get"
}

View file

@ -1,4 +1,4 @@
use httplz::{Event, Header, StatusLine};
use httplz::{Event, Header, RequestLine, StatusLine};
use crate::ToEvents;
@ -20,3 +20,26 @@ impl ToEvents for http::response::Parts {
out
}
}
impl ToEvents for http::request::Parts {
fn to_events(&self) -> Vec<Event> {
let mut out = vec![RequestLine {
version: httplz::Version::HTTP1_1,
method: httplz::Method::new_from_str(self.method.as_str()),
target: self
.uri
.path_and_query()
.map(|pq| pq.as_str())
.unwrap_or_else(|| self.uri.path()),
}
.into()];
out.extend(
self.headers
.iter()
.map(|(n, v)| Event::from(Header::from((n, v)))),
);
out
}
}

View file

@ -6,7 +6,7 @@
use std::{error::Error, io::Write, net::TcpListener};
use httplz::Lift;
use httplz::NeedsMoreData;
fn main() -> Result<(), Box<dyn Error>> {
let mut args = std::env::args().skip(1);
@ -21,38 +21,34 @@ fn main() -> Result<(), Box<dyn Error>> {
let mut conn = httplz::Connection::new(httplz::Role::Server);
let mut body: Vec<u8> = Vec::new();
let mut buf = vec![0; 1024].into_boxed_slice();
let mut buf = httplz_ext::Buf::new(&mut buf);
let mut buf = httplz::Buf::new(vec![0; 1024].into_boxed_slice());
let mut method_not_allowed = false;
loop {
let data = buf.filled();
let (remaining, r) = conn.poll_recv(data).lift();
match r.map_err(|e| e.kind) {
Err(httplz::ErrorKind::NeedMoreData) => {
buf.read_from(&mut stream)?;
continue;
},
Err(e) => panic!("{e:?}"),
Ok(event) => {
match event {
httplz::Event::RequestLine(r) => {
if !r.method.eq_ignore_ascii_case("post") {
method_not_allowed = true;
}
},
httplz::Event::RecvDone => break,
httplz::Event::BodyChunk(b) => body.extend_from_slice(b),
_ => (),
};
let r = conn.poll_recv(data);
if r.needs_more_data() {
buf.read_from(&mut stream)?;
continue;
}
let (amt, event) = r.unwrap();
match event {
httplz::Event::RequestLine(r) => {
if r.method != httplz::Method::Post {
method_not_allowed = true;
}
},
httplz::Event::RecvDone => break,
httplz::Event::BodyChunk(b) => body.extend_from_slice(b),
_ => (),
};
let len = data.len() - remaining.len();
buf.pop_front(len);
buf.pop_front(amt);
}
let body_len = format!("{}", body.len());
let parts: &[httplz::Event] = if method_not_allowed {
&[
httplz::Event::StatusLine(httplz::StatusLine {
@ -70,9 +66,11 @@ fn main() -> Result<(), Box<dyn Error>> {
status_code: 200,
status_text: "OK",
}),
httplz::Event::Header(httplz::Header::Special(
httplz::HeaderSpecial::ContentLength(body.len()),
)),
httplz::Header {
name: "Content-Length",
value: body_len.as_bytes(),
}
.into(),
httplz::Event::HeadersDone,
httplz::Event::BodyChunk(body.as_slice()),
httplz::Event::SendDone,

View file

@ -51,20 +51,12 @@ impl<'a> core::fmt::Display for Event<'a> {
Event::StatusLine(s) => {
write!(f, "{} {} {}", s.version, s.status_code, s.status_text)
},
Event::Header(h) => match h {
Header::Other { name, value } => {
write!(
f,
"{}: {}",
name,
core::str::from_utf8(value).unwrap_or("<binary>")
)
},
Header::ContentLength(cl) => write!(f, "Content-Length: {cl}"),
Header::TransferEncodingChunked => {
write!(f, "Transfer-Encoding: chunked")
},
},
Event::Header(Header { name, value }) => write!(
f,
"{}: {}",
name,
core::str::from_utf8(value).unwrap_or("<binary>")
),
Event::BodyChunk(b) => {
write!(f, "{}", core::str::from_utf8(b).unwrap_or("<binary>"))
},
@ -133,7 +125,7 @@ impl Connection {
let recv = match &self.state {
StateConnection::Recv(r) => r,
_ => {
return Err((bytes, ErrorKind::InvalidConnectionState.into()));
return Err(ErrorKind::InvalidConnectionState.into());
},
};
@ -155,7 +147,7 @@ impl Connection {
(_, StateRecv::Headers(body_state)) => {
if bytes.starts_with(b"\r\n") {
Ok((
&bytes[2..],
2,
(
Event::HeadersDone,
match body_state {
@ -166,11 +158,13 @@ impl Connection {
))
} else {
parse::header(bytes).map2(|h| {
let b = match h {
Header::TransferEncodingChunked => Some(BodyState::Chunked),
Header::ContentLength(c) => Some(BodyState::ContentLength(c)),
_ => *body_state,
};
let b = h
.special()
.map(|h| match h {
HeaderSpecial::TransferEncodingChunked => BodyState::Chunked,
HeaderSpecial::ContentLength(c) => BodyState::ContentLength(c),
})
.or(*body_state);
(
Event::Header(h),
@ -183,11 +177,11 @@ impl Connection {
(_, StateRecv::Body(body_state)) => match body_state {
BodyState::ContentLength(remaining) => {
if bytes.is_empty() && *remaining != 0 {
return fail(ErrorKind::NeedMoreData).tup(bytes);
return fail(ErrorKind::NeedMoreData);
}
if bytes.len() < *remaining {
Ok((
&[] as &[u8],
bytes.len(),
(
Event::BodyChunk(bytes),
StateConnection::Recv(StateRecv::Body(
@ -197,7 +191,7 @@ impl Connection {
))
} else {
Ok((
&bytes[*remaining..],
*remaining,
(
Event::BodyChunk(&bytes[..*remaining]),
StateConnection::Recv(StateRecv::ValidateDone),
@ -211,11 +205,11 @@ impl Connection {
(_, StateRecv::ValidateDone) => {
if bytes.is_empty() {
Ok((
bytes,
0,
(Event::RecvDone, StateConnection::Send(StateSend::StartLine)),
))
} else {
fail(ErrorKind::TrailingBytes).tup(bytes)
fail(ErrorKind::TrailingBytes)
}
},
};
@ -247,11 +241,13 @@ impl Connection {
(_, StateSend::Headers(body_state), Event::Header(h)) => {
write::header(h, w)?;
let bs = match h {
Header::Other { .. } => body_state,
Header::TransferEncodingChunked => Some(BodyState::Chunked),
Header::ContentLength(cl) => Some(BodyState::ContentLength(*cl)),
};
let bs = h
.special()
.map(|h| match h {
HeaderSpecial::TransferEncodingChunked => BodyState::Chunked,
HeaderSpecial::ContentLength(cl) => BodyState::ContentLength(cl),
})
.or(body_state);
Ok(StateConnection::Send(StateSend::Headers(bs)))
},

View file

@ -1,23 +1,14 @@
use crate::{
fail_details,
parts::{RequestLine, Version},
Error, ErrorKind, Header, Method, StatusLine, Tup,
Error, ErrorKind, Header, Method, StatusLine,
};
pub type Parse<'a, T> = Result<(&'a [u8], T), (&'a [u8], Error)>;
pub type Parse<'a, T> = Result<(usize, T), Error>;
pub trait NeedsMoreData {
fn needs_more_data(&self) -> bool;
}
impl<'a, T> NeedsMoreData for Parse<'a, T> {
fn needs_more_data(&self) -> bool {
self.as_ref()
.err()
.map(|e| e.1.kind)
.is_some_and(|e| e == ErrorKind::NeedMoreData)
}
}
impl<T> NeedsMoreData for Result<T, Error> {
fn needs_more_data(&self) -> bool {
self.as_ref()
.err()
@ -26,132 +17,126 @@ impl<T> NeedsMoreData for Result<T, Error> {
}
}
pub fn split_crlf(d: &[u8]) -> Option<(&[u8], &[u8])> {
pub fn split_crlf(d: &[u8]) -> Option<(&[u8], usize)> {
let p = d.windows(2).position(|w| w == b"\r\n")?;
Some((&d[..p], &d[p + 2..]))
Some((&d[..p], p + 2))
}
pub fn request_line(d: &[u8]) -> Parse<RequestLine> {
let (line, rest) = split_crlf(d).ok_or((d, ErrorKind::NeedMoreData.into()))?;
let (line, amt) = split_crlf(d).ok_or(Error::from(ErrorKind::NeedMoreData))?;
let go = || {
let mut it = line
.split(|b| b.is_ascii_whitespace())
.filter(|bs| !bs.is_empty());
let mut it = line
.split(|b| b.is_ascii_whitespace())
.filter(|bs| !bs.is_empty());
let (method, target, version) = match (it.next(), it.next(), it.next()) {
(Some(m), Some(t), Some(v)) => (m, t, v),
_ => {
return fail_details(
ErrorKind::Parse,
"request line doesn't have required number of elements",
);
},
};
let (method, target, version) = match (it.next(), it.next(), it.next()) {
(Some(m), Some(t), Some(v)) => (m, t, v),
_ => {
return fail_details(
ErrorKind::Parse,
"request line doesn't have required number of elements",
);
},
};
let method = match core::str::from_utf8(method) {
Ok(m) => m,
_ => {
return fail_details(ErrorKind::Parse, "expected method to be ascii");
},
};
let method = Method::new_from_str(method);
let method = match core::str::from_utf8(method) {
Ok(m) => m,
_ => {
return fail_details(ErrorKind::Parse, "expected method to be ascii");
},
};
let method = Method::new_from_str(method);
let target = match core::str::from_utf8(target) {
Ok(m) => m,
_ => {
return fail_details(ErrorKind::Parse, "expected target to be ascii");
},
};
let target = match core::str::from_utf8(target) {
Ok(m) => m,
_ => {
return fail_details(ErrorKind::Parse, "expected target to be ascii");
},
};
let version = match () {
_ if version.eq_ignore_ascii_case(b"http/1.1") => Version::HTTP1_1,
_ => {
return fail_details(ErrorKind::Parse, "unknown http version");
},
};
let version = match () {
_ if version.eq_ignore_ascii_case(b"http/1.1") => Version::HTTP1_1,
_ => {
return fail_details(ErrorKind::Parse, "unknown http version");
},
};
Ok(RequestLine {
Ok((
amt,
RequestLine {
method,
target,
version,
})
};
go().tup(rest)
},
))
}
pub fn status_line(d: &[u8]) -> Parse<StatusLine> {
let (line, rest) = split_crlf(d).ok_or((d, ErrorKind::NeedMoreData.into()))?;
let (line, amt) = split_crlf(d).ok_or(Error::from(ErrorKind::NeedMoreData))?;
let go = || {
let mut it = line
.split(|b| b.is_ascii_whitespace())
.filter(|bs| !bs.is_empty());
let mut it = line
.split(|b| b.is_ascii_whitespace())
.filter(|bs| !bs.is_empty());
let (version, status_code, status_text) = match (it.next(), it.next(), it.next()) {
(Some(m), Some(t), Some(v)) => (m, t, v),
_ => {
return fail_details(
ErrorKind::Parse,
"status line doesn't have required number of elements",
);
},
};
let (version, status_code, status_text) = match (it.next(), it.next(), it.next()) {
(Some(m), Some(t), Some(v)) => (m, t, v),
_ => {
return fail_details(
ErrorKind::Parse,
"status line doesn't have required number of elements",
);
},
};
let version = match () {
_ if version.eq_ignore_ascii_case(b"http/1.1") => Version::HTTP1_1,
_ => {
return fail_details(ErrorKind::Parse, "unknown http version");
},
};
let version = match () {
_ if version.eq_ignore_ascii_case(b"http/1.1") => Version::HTTP1_1,
_ => {
return fail_details(ErrorKind::Parse, "unknown http version");
},
};
let status_code = core::str::from_utf8(status_code)
.ok()
.and_then(|s| s.parse().ok())
.ok_or_else(|| Error::with_details(ErrorKind::Parse, "invalid status code"))?;
let status_code = core::str::from_utf8(status_code)
.ok()
.and_then(|s| s.parse().ok())
.ok_or_else(|| Error::with_details(ErrorKind::Parse, "invalid status code"))?;
let status_text = core::str::from_utf8(status_text)
.ok()
.ok_or_else(|| Error::with_details(ErrorKind::Parse, "invalid status text"))?;
let status_text = core::str::from_utf8(status_text)
.ok()
.ok_or_else(|| Error::with_details(ErrorKind::Parse, "invalid status text"))?;
Ok(StatusLine {
Ok((
amt,
StatusLine {
version,
status_code,
status_text,
})
};
go().tup(rest)
},
))
}
pub fn header(d: &[u8]) -> Parse<Header> {
let (line, rest) = split_crlf(d).ok_or((d, ErrorKind::NeedMoreData.into()))?;
let (line, amt) = split_crlf(d).ok_or(Error::from(ErrorKind::NeedMoreData))?;
let go = || {
let mut it = line.split(|b| *b == b':').filter(|bs| !bs.is_empty());
let mut it = line.split(|b| *b == b':').filter(|bs| !bs.is_empty());
let (name, value) = match (it.next(), it.next()) {
(Some(n), Some(v)) => (n, v),
_ => {
return fail_details(
ErrorKind::Parse,
"header doesn't have required number of elements",
);
},
};
let name = match core::str::from_utf8(name) {
Ok(m) => m,
_ => return fail_details(ErrorKind::Parse, "expected target to be ascii"),
};
let name = name.trim();
let value = value.trim_ascii();
Ok(Header::from((name, value)))
let (name, value) = match (it.next(), it.next()) {
(Some(n), Some(v)) => (n, v),
_ => {
return fail_details(
ErrorKind::Parse,
"header doesn't have required number of elements",
);
},
};
go().tup(rest)
let name = match core::str::from_utf8(name) {
Ok(m) => m,
_ => return fail_details(ErrorKind::Parse, "expected target to be ascii"),
};
let name = name.trim();
let value = value.trim_ascii();
Ok((amt, Header::from((name, value))))
}

View file

@ -76,10 +76,38 @@ pub struct StatusLine<'a> {
}
#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
pub enum Header<'a> {
pub struct Header<'a> {
pub name: &'a str,
pub value: &'a [u8],
}
impl<'a> Header<'a> {
pub fn special(&self) -> Option<HeaderSpecial> {
let Self { name, value } = self;
match () {
_ if name.eq_ignore_ascii_case("transfer-encoding")
&& value.eq_ignore_ascii_case(b"chunked") =>
{
Some(HeaderSpecial::TransferEncodingChunked)
},
_ if name.eq_ignore_ascii_case("content-length") => {
match core::str::from_utf8(value)
.ok()
.and_then(|s| s.parse().ok())
{
Some(v) => Some(HeaderSpecial::ContentLength(v)),
_ => None,
}
},
_ => None,
}
}
}
#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
pub enum HeaderSpecial {
TransferEncodingChunked,
ContentLength(usize),
Other { name: &'a str, value: &'a [u8] },
}
impl<'a, Name, Value> From<(&'a Name, &'a Value)> for Header<'a>
@ -87,27 +115,10 @@ where
Name: AsRef<str> + ?Sized,
Value: AsRef<[u8]> + ?Sized,
{
fn from(value: (&'a Name, &'a Value)) -> Self {
let (name, value) = value;
let name = name.as_ref();
let value = value.as_ref();
match () {
_ if name.eq_ignore_ascii_case("transfer-encoding")
&& value.eq_ignore_ascii_case(b"chunked") =>
{
Header::TransferEncodingChunked
},
_ if name.eq_ignore_ascii_case("content-length") => {
match core::str::from_utf8(value)
.ok()
.and_then(|s| s.parse().ok())
{
Some(v) => Header::ContentLength(v),
_ => Header::Other { name, value },
}
},
_ => Header::Other { name, value },
fn from((name, value): (&'a Name, &'a Value)) -> Self {
Header {
name: name.as_ref(),
value: value.as_ref(),
}
}
}

View file

@ -1,37 +1,13 @@
use crate::{ErrorKind, Write, Written};
pub trait ResultTupExt<A, T, B, E> {
fn map2<U>(self, f: impl FnOnce(T) -> U) -> Result<(A, U), (B, E)>;
pub trait ResultTupExt<A, T, E> {
fn map2<U>(self, f: impl FnOnce(T) -> U) -> Result<(A, U), E>;
}
impl<A, T, B, E> ResultTupExt<A, T, B, E> for Result<(A, T), (B, E)> {
fn map2<U>(self, f: impl FnOnce(T) -> U) -> Result<(A, U), (B, E)> {
impl<A, T, E> ResultTupExt<A, T, E> for Result<(A, T), E> {
fn map2<U>(self, f: impl FnOnce(T) -> U) -> Result<(A, U), E> {
match self {
Ok((a, t)) => Ok((a, f(t))),
Err((b, e)) => Err((b, e)),
}
}
}
pub trait Tup<T, E> {
fn tup<V>(self, v: V) -> Result<(V, T), (V, E)>;
}
impl<T, E> Tup<T, E> for Result<T, E> {
fn tup<V>(self, v: V) -> Result<(V, T), (V, E)> {
match self {
Ok(t) => Ok((v, t)),
Err(e) => Err((v, e)),
}
}
}
pub trait Lift<V, T, E> {
fn lift(self) -> (V, Result<T, E>);
}
impl<V, T, E> Lift<V, T, E> for Result<(V, T), (V, E)> {
fn lift(self) -> (V, Result<T, E>) {
match self {
Ok((v, t)) => (v, Ok(t)),
Err((v, e)) => (v, Err(e)),
Err(e) => Err(e),
}
}
}

View file

@ -107,14 +107,9 @@ pub fn request_line(rl: &RequestLine, mut w: impl Write) -> Written {
}
pub fn header(h: &Header, mut w: impl Write) -> Written {
match h {
Header::TransferEncodingChunked => write!(w, "Transfer-Encoding: Chunked"),
Header::ContentLength(cl) => write!(w, "Content-Length: {}", cl),
Header::Other { name, value } => {
write!(w, "{name}: ")?;
w.write(value)
},
}?;
let Header { name, value } = h;
write!(w, "{name}: ")?;
w.write(value)?;
write!(w, "\r\n")
}