Efficient, zero-buffer streaming for large HTTP payloads — built on top of net/http.

go get github.com/nativebpm/httpstream

httpstream provides a minimal, streaming-oriented API for building HTTP requests without buffering entire payloads in memory.
Ideal for large JSON bodies, multipart uploads, generated archives, or continuous data feeds.

Key Features

Stream data directly via io.Pipe—no intermediate buffers
Constant memory usage (O(1)), regardless of payload size
Natural backpressure (writes block when receiver is slow)
Thin net/http wrapper—fully compatible
Middleware support: func(http.RoundTripper) http.RoundTripper
Fluent API for readability (GET, POST, Multipart, etc.)
No goroutine leaks, no globals

How It Works

httpstream connects your writer directly to the HTTP transport. Data is transmitted as it's produced, allowing the server to start processing immediately—without waiting for the full body to be buffered.

Why Streaming Matters

Traditional HTTP clients buffer request bodies entirely before sending. For large or dynamically generated payloads, this can lead to:
High memory usage (O(n) where n = payload size)
Slow transmission start (server waits for full upload)
Out-of-memory errors in constrained environments

httpstream eliminates these issues by design.

Simple Example With Logger

package main

import (
 "context"
 "log"
 "log/slog"
 "net/http"
 "os"
 "time"

 "github.com/nativebpm/httpstream"
)

func main() {

 logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
  Level: slog.LevelInfo,
 }))

 loggingMiddleware := httpstream.LoggingMiddleware(logger)

 client, err := httpstream.NewClient(&http.Client{Timeout: 10 * time.Second}, "https://httpbin.org")
 if err != nil {
  log.Fatal(err)
 }

 resp, err := client.GET(context.Background(), "/get").
  Use(loggingMiddleware).
  Send()
 if err != nil {
  log.Fatal(err)
 }
 resp.Body.Close()
}

Multipart Streaming Example

This example demonstrates streaming data between two servers using httpstream.

Server 1 (:8080) — Generating a Large File

package main

import (
 "fmt"
 "io"
 "net/http"
 "strings"
)

func main() {
 http.HandleFunc("/file", func(w http.ResponseWriter, r *http.Request) {
  w.Header().Set("Content-Type", "text/plain")
  w.Header().Set("Content-Disposition", "attachment; filename=large.txt")

  var builder strings.Builder
  for i := 1; i <= 10000000; i++ {
   builder.WriteString(fmt.Sprintf("Line %d: This is a line in the large file.\n", i))
  }
  reader := strings.NewReader(builder.String())
  _, err := io.Copy(w, reader)
  if err != nil {
   http.Error(w, "Failed to generate file", http.StatusInternalServerError)
  }
 })

 fmt.Println("Server 1 running on :8080")
 http.ListenAndServe(":8080", nil)
}

Server 2 (:8081) — Receiving Multipart Upload

package main

import (
 "fmt"
 "io"
 "log/slog"
 "net/http"
 "os"
)

func main() {
 logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}))

 http.HandleFunc("/upload", func(w http.ResponseWriter, r *http.Request) {
  err := r.ParseMultipartForm(32 << 20) // 32MB max
  if err != nil {
   logger.Error("Failed to parse multipart", "error", err)
   http.Error(w, err.Error(), http.StatusBadRequest)
   return
  }

  file, header, err := r.FormFile("file")
  if err != nil {
   logger.Error("Failed to get form file", "error", err)
   http.Error(w, err.Error(), http.StatusBadRequest)
   return
  }
  defer file.Close()

  dst, err := os.Create(header.Filename)
  if err != nil {
   logger.Error("Failed to create file", "error", err)
   http.Error(w, err.Error(), http.StatusInternalServerError)
   return
  }
  defer dst.Close()

  _, err = io.Copy(dst, file)
  if err != nil {
   logger.Error("Failed to copy file", "error", err)
   http.Error(w, err.Error(), http.StatusInternalServerError)
   return
  }

  logger.Info("File saved successfully", "filename", header.Filename)
  fmt.Fprintf(w, "File %s uploaded and saved", header.Filename)
 })

 fmt.Println("Server 2 running on :8081")
 http.ListenAndServe(":8081", nil)
}

Client — Streaming File from Server 1 to Server 2

package main

import (
 "context"
 "fmt"
 "io"
 "log/slog"
 "mime"
 "net/http"
 "runtime"
 "time"

 "github.com/nativebpm/httpstream"
)

type countingReader struct {
 reader io.Reader
 count  int64
}

func (cr *countingReader) Read(p []byte) (n int, err error) {
 n, err = cr.reader.Read(p)
 cr.count += int64(n)
 return n, err
}

func (cr *countingReader) Close() error {
 if closer, ok := cr.reader.(io.Closer); ok {
  return closer.Close()
 }
 return nil
}

func main() {
 logger := slog.Default()
 var m runtime.MemStats
 runtime.ReadMemStats(&m)
 logger.Info("Before streaming", "Alloc (KB)", m.Alloc/1024, "TotalAlloc (KB)", m.TotalAlloc/1024)

 client := &http.Client{Timeout: 60 * time.Second}

 server1Client, _ := httpstream.NewClient(client, "http://localhost:8080")
 server1Client.Use(httpstream.LoggingMiddleware(logger.WithGroup("server1")))

 server2Client, _ := httpstream.NewClient(client, "http://localhost:8081")
 server2Client.Use(httpstream.LoggingMiddleware(logger.WithGroup("server2")))

 server1Resp, _ := server1Client.GET(context.Background(), "/file").Timeout(30 * time.Second).Send()
 defer server1Resp.Body.Close()

 filename := filename(server1Resp.Header, "default_filename")
 counter := &countingReader{reader: server1Resp.Body}

 server2Resp, _ := server2Client.Multipart(context.Background(), "/upload").
  File("file", filename, counter).
  Timeout(30 * time.Second).
  Send()
 defer server2Resp.Body.Close()

 runtime.ReadMemStats(&m)
 slog.Info("After streaming", "Alloc (KB)", m.Alloc/1024, "TotalAlloc (KB)", m.TotalAlloc/1024)

 streamedMB := float64(counter.count) / (1024 * 1024)
 slog.Info("Data streamed through pipeline", "bytes", counter.count, "megabytes", fmt.Sprintf("%.2f MB", streamedMB))
}

func filename(headers http.Header, defaultName string) string {
 if v := headers.Get("Content-Disposition"); v != "" {
  _, params, err := mime.ParseMediaType(v)
  if err == nil {
   if fn, ok := params["filename"]; ok {
    return fn
   }
  }
 }
 return defaultName
}

Running the Example

Run the servers and client in separate terminals:

go run server1/main.go
go run server2/main.go
go run main.go

Logs show memory usage before and after streaming and total streamed data.

Before streaming "Alloc (KB)"=218 "TotalAlloc (KB)"=218
Sending request server1.method=GET server1.url=http://localhost:8080/file
Response received server1.status=200 server1.duration=1.849253692s
Sending request server2.method=POST server2.url=http://localhost:8081/upload
Response received server2.status=200 server2.duration=4.204263582s
After streaming "Alloc (KB)"=694 "TotalAlloc (KB)"=694
Data streamed through pipeline bytes=478888897 megabytes="456.70 MB"
Upload successful "server2Resp response"="File large.txt uploaded and save"

Result

Server 1 generates a large file with numbered lines.
Client streams the file from Server 1 to Server 2 without buffering it in memory.
Server 2 saves the file locally.
Logs show upload confirmation and memory usage.

Streaming is efficient, constant in memory, and ready for large payloads.

Full Examples

Github: https://github.com/nativebpm/httpstream

Stream-first HTTP Client for Go