soup/httplz
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Fetchplz!

Browse source
This commit is contained in:
soup 2024-10-18 00:29:46 -04:00
parent 523bbd2e4c
commit da94c17fe8
No known key found for this signature in database
12 changed files with 426 additions and 226 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
Cargo.lock generated
View file

@ -8,6 +8,15 @@ version = "1.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3"
[[package]]
name = "fetchplz"
version = "0.0.0"
dependencies = [
"http",
"httplz",
"httplzx",
]
[[package]]
name = "fnv"
version = "1.0.7"

6
Cargo.toml
View file

@ -7,9 +7,15 @@ description = "A sans-io, no-std HTTP implementation"
[workspace]
members = [
"crates/fetchplz",
"crates/httplzx"
]
[workspace.dependencies]
httplz = { path = ".", features = ["std"] }
httplzx = { path = "./crates/httplzx" }
fetchplz = { path = "./crates/fetchplz" }
[features]
std = []

9
crates/fetchplz/Cargo.toml Normal file
View file

@ -0,0 +1,9 @@
[package]
name = "fetchplz"
version = "0.0.0"
edition = "2021"
[dependencies]
httplz = { workspace = true }
httplzx = { workspace = true }
http = { version = "1.1.0" }

181
crates/fetchplz/src/lib.rs Normal file
View file

@ -0,0 +1,181 @@
use std::io::{Error as IOError, Read, Write};
use std::net::TcpStream;
use std::str::FromStr;
use httplz::NeedsMoreData;
use httplzx::ToEvents;
pub struct Body {
buf: httplz::Buf<Box<[u8]>, u8>,
chunk_end: usize,
read_amt: usize,
stream: TcpStream,
conn: httplz::Connection,
}
impl Body {
fn find_next_chunk_sz(&mut self) -> Result<(), IOError> {
self.buf.pop_front(self.chunk_end);
self.chunk_end = 0;
self.read_amt = 0;
while self.conn.is_recving() {
dbg!(self.conn, String::from_utf8_lossy(self.buf.filled()));
let res = self.conn.poll_recv(self.buf.filled());
if res.needs_more_data() {
self.buf.read_from(&mut self.stream)?;
continue;
}
let (amt, ev) = res.map_err(|e| IOError::other(format!("{e:?}")))?;
match ev {
httplz::Event::BodyChunk(b) => {
self.chunk_end = b.len();
return Ok(());
},
_ => self.buf.pop_front(amt),
};
}
Ok(())
}
}
impl Read for Body {
fn read(&mut self, mut buf: &mut [u8]) -> std::io::Result<usize> {
if self.chunk_end == self.read_amt {
self.find_next_chunk_sz()?;
}
let chunk = &self.buf.filled()[self.read_amt..self.chunk_end];
let amt = buf.write(chunk)?;
self.read_amt += amt;
Ok(amt)
}
}
impl Body {
pub fn collect_bytes(self) -> Result<Vec<u8>, IOError> {
self.bytes().collect()
}
}
pub type Response = http::Response<Body>;
#[derive(Debug)]
pub enum Error {
IOError(IOError),
HttplzError(httplz::Error),
InvalidUri,
InvalidStatusCode,
InvalidHeaderValue,
}
impl From<IOError> for Error {
fn from(value: IOError) -> Self {
Self::IOError(value)
}
}
impl From<httplz::Error> for Error {
fn from(value: httplz::Error) -> Self {
Self::HttplzError(value)
}
}
pub fn fetch(path: &str) -> Result<Response, Error> {
let uri = http::Uri::from_str(path).map_err(|_| Error::InvalidUri)?;
let scheme = uri.scheme().cloned().unwrap_or(http::uri::Scheme::HTTPS);
let host = uri.host().ok_or(Error::InvalidUri)?;
let mut stream = TcpStream::connect(format!(
"{}:{}",
host,
uri.port_u16().unwrap_or_else(|| match scheme.as_str() {
"http" => 80,
_ => 443,
})
))?;
let mut conn = httplz::Connection::new(httplz::Role::Client);
let (mut parts, _) = http::Request::new(()).into_parts();
parts
.headers
.insert("Host", http::HeaderValue::from_str(host).unwrap());
parts.uri = uri;
let mut events = parts.to_events();
events.extend_from_slice(&[httplz::Event::HeadersDone, httplz::Event::SendDone]);
dbg!("a");
let mut buf = httplz::Buf::new(vec![0; 4096].into_boxed_slice());
for event in events {
conn.handle_send(&event, &mut buf)?;
}
stream.write_all(buf.filled())?;
assert!(conn.is_recving());
buf.clear();
let (mut parts, _) = http::Response::new(()).into_parts();
while conn.is_recving() {
let data = buf.filled();
let res = conn.poll_recv(data);
if res.needs_more_data() {
buf.read_from(&mut stream)?;
continue;
}
let (amt, event) = res?;
let mut brk = false;
match event {
httplz::Event::StatusLine(sl) => {
parts.status = http::StatusCode::from_u16(sl.status_code)
.map_err(|_| Error::InvalidStatusCode)?;
parts.version = http::Version::HTTP_11;
},
httplz::Event::Header(h) => {
parts.headers.insert(
http::HeaderName::from_str(h.name).unwrap(),
http::HeaderValue::from_bytes(h.value)
.map_err(|_| Error::InvalidHeaderValue)?,
);
},
httplz::Event::HeadersDone => brk = true,
_ => (),
};
dbg!(amt, brk);
buf.pop_front(amt);
if brk {
break;
}
}
Ok(http::Response::from_parts(
parts,
Body {
buf,
stream,
conn,
chunk_end: 0,
read_amt: 0,
},
))
}
#[cfg(test)]
mod test_fetch {
use crate::fetch;
#[test]
fn test_fetch_1() {
let resp = fetch("http://httpbin.org/get")
.unwrap()
.map(|b| b.collect_bytes().unwrap());
let body = resp.into_body();
let body = String::from_utf8_lossy(&body);
assert_eq!(
body,
include_str!("../testing/snapshots/httpbin_get_empty.txt")
)
}
}

11
crates/fetchplz/testing/snapshots/httpbin_get_empty.txt Normal file
View file

@ -0,0 +1,11 @@
{
"args": {},
"headers": {
"Accept": "*/*",
"Host": "httpbin.org",
"User-Agent": "curl/8.7.1",
"X-Amzn-Trace-Id": "Root=1-6711c17e-1fdd277564daedff276ec60a"
},
"origin": "72.94.94.124",
"url": "https://httpbin.org/get"
}

25
crates/httplzx/src/compat_http.rs
View file

@ -1,4 +1,4 @@
use httplz::{Event, Header, StatusLine};
use httplz::{Event, Header, RequestLine, StatusLine};
use crate::ToEvents;
@ -20,3 +20,26 @@ impl ToEvents for http::response::Parts {
out
}
}
impl ToEvents for http::request::Parts {
fn to_events(&self) -> Vec<Event> {
let mut out = vec![RequestLine {
version: httplz::Version::HTTP1_1,
method: httplz::Method::new_from_str(self.method.as_str()),
target: self
.uri
.path_and_query()
.map(|pq| pq.as_str())
.unwrap_or_else(|| self.uri.path()),
}
.into()];
out.extend(
self.headers
.iter()
.map(|(n, v)| Event::from(Header::from((n, v)))),
);
out
}
}

32
examples/echo.rs
View file

@ -6,7 +6,7 @@
use std::{error::Error, io::Write, net::TcpListener};
use httplz::Lift;
use httplz::NeedsMoreData;
fn main() -> Result<(), Box<dyn Error>> {
let mut args = std::env::args().skip(1);
@ -21,24 +21,22 @@ fn main() -> Result<(), Box<dyn Error>> {
let mut conn = httplz::Connection::new(httplz::Role::Server);
let mut body: Vec<u8> = Vec::new();
let mut buf = vec![0; 1024].into_boxed_slice();
let mut buf = httplz_ext::Buf::new(&mut buf);
let mut buf = httplz::Buf::new(vec![0; 1024].into_boxed_slice());
let mut method_not_allowed = false;
loop {
let data = buf.filled();
let (remaining, r) = conn.poll_recv(data).lift();
match r.map_err(|e| e.kind) {
Err(httplz::ErrorKind::NeedMoreData) => {
let r = conn.poll_recv(data);
if r.needs_more_data() {
buf.read_from(&mut stream)?;
continue;
},
Err(e) => panic!("{e:?}"),
Ok(event) => {
}
let (amt, event) = r.unwrap();
match event {
httplz::Event::RequestLine(r) => {
if !r.method.eq_ignore_ascii_case("post") {
if r.method != httplz::Method::Post {
method_not_allowed = true;
}
},
@ -46,13 +44,11 @@ fn main() -> Result<(), Box<dyn Error>> {
httplz::Event::BodyChunk(b) => body.extend_from_slice(b),
_ => (),
};
},
};
let len = data.len() - remaining.len();
buf.pop_front(len);
buf.pop_front(amt);
}
let body_len = format!("{}", body.len());
let parts: &[httplz::Event] = if method_not_allowed {
&[
httplz::Event::StatusLine(httplz::StatusLine {
@ -70,9 +66,11 @@ fn main() -> Result<(), Box<dyn Error>> {
status_code: 200,
status_text: "OK",
}),
httplz::Event::Header(httplz::Header::Special(
httplz::HeaderSpecial::ContentLength(body.len()),
)),
httplz::Header {
name: "Content-Length",
value: body_len.as_bytes(),
}
.into(),
httplz::Event::HeadersDone,
httplz::Event::BodyChunk(body.as_slice()),
httplz::Event::SendDone,

50
src/lib.rs
View file

@ -51,20 +51,12 @@ impl<'a> core::fmt::Display for Event<'a> {
Event::StatusLine(s) => {
write!(f, "{} {} {}", s.version, s.status_code, s.status_text)
},
Event::Header(h) => match h {
Header::Other { name, value } => {
write!(
Event::Header(Header { name, value }) => write!(
f,
"{}: {}",
name,
core::str::from_utf8(value).unwrap_or("<binary>")
)
},
Header::ContentLength(cl) => write!(f, "Content-Length: {cl}"),
Header::TransferEncodingChunked => {
write!(f, "Transfer-Encoding: chunked")
},
},
),
Event::BodyChunk(b) => {
write!(f, "{}", core::str::from_utf8(b).unwrap_or("<binary>"))
},
@ -133,7 +125,7 @@ impl Connection {
let recv = match &self.state {
StateConnection::Recv(r) => r,
_ => {
return Err((bytes, ErrorKind::InvalidConnectionState.into()));
return Err(ErrorKind::InvalidConnectionState.into());
},
};
@ -155,7 +147,7 @@ impl Connection {
(_, StateRecv::Headers(body_state)) => {
if bytes.starts_with(b"\r\n") {
Ok((
&bytes[2..],
2,
(
Event::HeadersDone,
match body_state {
@ -166,11 +158,13 @@ impl Connection {
))
} else {
parse::header(bytes).map2(|h| {
let b = match h {
Header::TransferEncodingChunked => Some(BodyState::Chunked),
Header::ContentLength(c) => Some(BodyState::ContentLength(c)),
_ => *body_state,
};
let b = h
.special()
.map(|h| match h {
HeaderSpecial::TransferEncodingChunked => BodyState::Chunked,
HeaderSpecial::ContentLength(c) => BodyState::ContentLength(c),
})
.or(*body_state);
(
Event::Header(h),
@ -183,11 +177,11 @@ impl Connection {
(_, StateRecv::Body(body_state)) => match body_state {
BodyState::ContentLength(remaining) => {
if bytes.is_empty() && *remaining != 0 {
return fail(ErrorKind::NeedMoreData).tup(bytes);
return fail(ErrorKind::NeedMoreData);
}
if bytes.len() < *remaining {
Ok((
&[] as &[u8],
bytes.len(),
(
Event::BodyChunk(bytes),
StateConnection::Recv(StateRecv::Body(
@ -197,7 +191,7 @@ impl Connection {
))
} else {
Ok((
&bytes[*remaining..],
*remaining,
(
Event::BodyChunk(&bytes[..*remaining]),
StateConnection::Recv(StateRecv::ValidateDone),
@ -211,11 +205,11 @@ impl Connection {
(_, StateRecv::ValidateDone) => {
if bytes.is_empty() {
Ok((
bytes,
0,
(Event::RecvDone, StateConnection::Send(StateSend::StartLine)),
))
} else {
fail(ErrorKind::TrailingBytes).tup(bytes)
fail(ErrorKind::TrailingBytes)
}
},
};
@ -247,11 +241,13 @@ impl Connection {
(_, StateSend::Headers(body_state), Event::Header(h)) => {
write::header(h, w)?;
let bs = match h {
Header::Other { .. } => body_state,
Header::TransferEncodingChunked => Some(BodyState::Chunked),
Header::ContentLength(cl) => Some(BodyState::ContentLength(*cl)),
};
let bs = h
.special()
.map(|h| match h {
HeaderSpecial::TransferEncodingChunked => BodyState::Chunked,
HeaderSpecial::ContentLength(cl) => BodyState::ContentLength(cl),
})
.or(body_state);
Ok(StateConnection::Send(StateSend::Headers(bs)))
},

51
src/parse.rs
View file

@ -1,23 +1,14 @@
use crate::{
fail_details,
parts::{RequestLine, Version},
Error, ErrorKind, Header, Method, StatusLine, Tup,
Error, ErrorKind, Header, Method, StatusLine,
};
pub type Parse<'a, T> = Result<(&'a [u8], T), (&'a [u8], Error)>;
pub type Parse<'a, T> = Result<(usize, T), Error>;
pub trait NeedsMoreData {
fn needs_more_data(&self) -> bool;
}
impl<'a, T> NeedsMoreData for Parse<'a, T> {
fn needs_more_data(&self) -> bool {
self.as_ref()
.err()
.map(|e| e.1.kind)
.is_some_and(|e| e == ErrorKind::NeedMoreData)
}
}
impl<T> NeedsMoreData for Result<T, Error> {
fn needs_more_data(&self) -> bool {
self.as_ref()
.err()
@ -26,16 +17,15 @@ impl<T> NeedsMoreData for Result<T, Error> {
}
}
pub fn split_crlf(d: &[u8]) -> Option<(&[u8], &[u8])> {
pub fn split_crlf(d: &[u8]) -> Option<(&[u8], usize)> {
let p = d.windows(2).position(|w| w == b"\r\n")?;
Some((&d[..p], &d[p + 2..]))
Some((&d[..p], p + 2))
}
pub fn request_line(d: &[u8]) -> Parse<RequestLine> {
let (line, rest) = split_crlf(d).ok_or((d, ErrorKind::NeedMoreData.into()))?;
let (line, amt) = split_crlf(d).ok_or(Error::from(ErrorKind::NeedMoreData))?;
let go = || {
let mut it = line
.split(|b| b.is_ascii_whitespace())
.filter(|bs| !bs.is_empty());
@ -72,20 +62,19 @@ pub fn request_line(d: &[u8]) -> Parse<RequestLine> {
},
};
Ok(RequestLine {
Ok((
amt,
RequestLine {
method,
target,
version,
})
};
go().tup(rest)
},
))
}
pub fn status_line(d: &[u8]) -> Parse<StatusLine> {
let (line, rest) = split_crlf(d).ok_or((d, ErrorKind::NeedMoreData.into()))?;
let (line, amt) = split_crlf(d).ok_or(Error::from(ErrorKind::NeedMoreData))?;
let go = || {
let mut it = line
.split(|b| b.is_ascii_whitespace())
.filter(|bs| !bs.is_empty());
@ -116,20 +105,19 @@ pub fn status_line(d: &[u8]) -> Parse<StatusLine> {
.ok()
.ok_or_else(|| Error::with_details(ErrorKind::Parse, "invalid status text"))?;
Ok(StatusLine {
Ok((
amt,
StatusLine {
version,
status_code,
status_text,
})
};
go().tup(rest)
},
))
}
pub fn header(d: &[u8]) -> Parse<Header> {
let (line, rest) = split_crlf(d).ok_or((d, ErrorKind::NeedMoreData.into()))?;
let (line, amt) = split_crlf(d).ok_or(Error::from(ErrorKind::NeedMoreData))?;
let go = || {
let mut it = line.split(|b| *b == b':').filter(|bs| !bs.is_empty());
let (name, value) = match (it.next(), it.next()) {
@ -150,8 +138,5 @@ pub fn header(d: &[u8]) -> Parse<Header> {
let value = value.trim_ascii();
Ok(Header::from((name, value)))
};
go().tup(rest)
Ok((amt, Header::from((name, value))))
}

57
src/parts.rs
View file

@ -76,10 +76,38 @@ pub struct StatusLine<'a> {
}
#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
pub enum Header<'a> {
pub struct Header<'a> {
pub name: &'a str,
pub value: &'a [u8],
}
impl<'a> Header<'a> {
pub fn special(&self) -> Option<HeaderSpecial> {
let Self { name, value } = self;
match () {
_ if name.eq_ignore_ascii_case("transfer-encoding")
&& value.eq_ignore_ascii_case(b"chunked") =>
{
Some(HeaderSpecial::TransferEncodingChunked)
},
_ if name.eq_ignore_ascii_case("content-length") => {
match core::str::from_utf8(value)
.ok()
.and_then(|s| s.parse().ok())
{
Some(v) => Some(HeaderSpecial::ContentLength(v)),
_ => None,
}
},
_ => None,
}
}
}
#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
pub enum HeaderSpecial {
TransferEncodingChunked,
ContentLength(usize),
Other { name: &'a str, value: &'a [u8] },
}
impl<'a, Name, Value> From<(&'a Name, &'a Value)> for Header<'a>
@ -87,27 +115,10 @@ where
Name: AsRef<str> + ?Sized,
Value: AsRef<[u8]> + ?Sized,
{
fn from(value: (&'a Name, &'a Value)) -> Self {
let (name, value) = value;
let name = name.as_ref();
let value = value.as_ref();
match () {
_ if name.eq_ignore_ascii_case("transfer-encoding")
&& value.eq_ignore_ascii_case(b"chunked") =>
{
Header::TransferEncodingChunked
},
_ if name.eq_ignore_ascii_case("content-length") => {
match core::str::from_utf8(value)
.ok()
.and_then(|s| s.parse().ok())
{
Some(v) => Header::ContentLength(v),
_ => Header::Other { name, value },
}
},
_ => Header::Other { name, value },
fn from((name, value): (&'a Name, &'a Value)) -> Self {
Header {
name: name.as_ref(),
value: value.as_ref(),
}
}
}

34
src/util.rs
View file

@ -1,37 +1,13 @@
use crate::{ErrorKind, Write, Written};
pub trait ResultTupExt<A, T, B, E> {
fn map2<U>(self, f: impl FnOnce(T) -> U) -> Result<(A, U), (B, E)>;
pub trait ResultTupExt<A, T, E> {
fn map2<U>(self, f: impl FnOnce(T) -> U) -> Result<(A, U), E>;
}
impl<A, T, B, E> ResultTupExt<A, T, B, E> for Result<(A, T), (B, E)> {
fn map2<U>(self, f: impl FnOnce(T) -> U) -> Result<(A, U), (B, E)> {
impl<A, T, E> ResultTupExt<A, T, E> for Result<(A, T), E> {
fn map2<U>(self, f: impl FnOnce(T) -> U) -> Result<(A, U), E> {
match self {
Ok((a, t)) => Ok((a, f(t))),
Err((b, e)) => Err((b, e)),
}
}
}
pub trait Tup<T, E> {
fn tup<V>(self, v: V) -> Result<(V, T), (V, E)>;
}
impl<T, E> Tup<T, E> for Result<T, E> {
fn tup<V>(self, v: V) -> Result<(V, T), (V, E)> {
match self {
Ok(t) => Ok((v, t)),
Err(e) => Err((v, e)),
}
}
}
pub trait Lift<V, T, E> {
fn lift(self) -> (V, Result<T, E>);
}
impl<V, T, E> Lift<V, T, E> for Result<(V, T), (V, E)> {
fn lift(self) -> (V, Result<T, E>) {
match self {
Ok((v, t)) => (v, Ok(t)),
Err((v, e)) => (v, Err(e)),
Err(e) => Err(e),
}
}
}

9
src/write.rs
View file

@ -107,14 +107,9 @@ pub fn request_line(rl: &RequestLine, mut w: impl Write) -> Written {
}
pub fn header(h: &Header, mut w: impl Write) -> Written {
match h {
Header::TransferEncodingChunked => write!(w, "Transfer-Encoding: Chunked"),
Header::ContentLength(cl) => write!(w, "Content-Length: {}", cl),
Header::Other { name, value } => {
let Header { name, value } = h;
write!(w, "{name}: ")?;
w.write(value)
},
}?;
w.write(value)?;
write!(w, "\r\n")
}