first commit

This commit is contained in:
William Valentin
2025-08-15 23:45:13 -07:00
commit c167c8623c
29 changed files with 1971 additions and 0 deletions

1
README.txt Normal file
View File

@@ -0,0 +1 @@
These are two different takes to stream audio ;)

View File

@@ -0,0 +1,34 @@
# Dockerfile
FROM alpine:3.20
LABEL maintainer="will"
LABEL version="1.0"
# ffmpeg + bash + coreutils
RUN apk add --no-cache ffmpeg bash coreutils procps
WORKDIR /app
# Where optional URL files can be mounted
VOLUME ["/app"]
COPY entrypoint.sh /usr/local/bin/entrypoint.sh
RUN chmod +x /usr/local/bin/entrypoint.sh
# Defaults (override at runtime)
ENV URLS="" \
URLS_FILE="/app/urls.txt" \
LOOP="1" \
PROTOCOL="udp" \
TARGET="udp://239.0.0.1:1234?ttl=16" \
# m4a is typically AAC-in-MP4; copy when target supports AAC:
CODEC="aac" \
COPY_CODEC_WHEN_POSSIBLE="1" \
BITRATE="160k" \
SAMPLE_RATE="48000" \
FFMPEG_EXTRA_ARGS=""
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \
CMD test -f /tmp/ffmpeg.pid && ps -p "$(cat /tmp/ffmpeg.pid)" > /dev/null

View File

@@ -0,0 +1 @@
Audio files are copied from minio/S3, then streamed via ffmpeg!

View File

@@ -0,0 +1,126 @@
#!/usr/bin/env bash
set -euo pipefail
# --- Configuration (from environment variables with defaults) ---
URLS="${URLS:-}"
URLS_FILE="${URLS_FILE:-/app/urls.txt}"
LOOP="${LOOP:-1}"
PROTOCOL="${PROTOCOL:-udp}" # udp | rtp | rtmp | icecast
TARGET="${TARGET:-udp://239.0.0.1:1234?ttl=16}"
CODEC="${CODEC:-aac}"
COPY_CODEC_WHEN_POSSIBLE="${COPY_CODEC_WHEN_POSSIBLE:-1}"
BITRATE="${BITRATE:-160k}"
SAMPLE_RATE="${SAMPLE_RATE:-48000}"
FFMPEG_EXTRA_ARGS="${FFMPEG_EXTRA_ARGS:-}"
# --- Globals ---
PID_FILE="/tmp/ffmpeg.pid"
PLAYLIST_FILE="/tmp/playlist.txt"
declare -a URL_LIST=()
declare -a CODEC_ARGS=()
declare -a OUTPUT_FORMAT_ARGS=()
# --- Functions ---
# Graceful cleanup on exit
cleanup() {
rm -f "$PID_FILE" "$PLAYLIST_FILE"
}
trap cleanup EXIT
# Populates URL_LIST from environment or file
get_urls() {
local raw_urls=()
if [[ -n "$URLS" ]]; then
IFS=$'\n,' read -r -d '' -a raw_urls < <(printf '%s\0' "$URLS")
elif [[ -f "$URLS_FILE" ]]; then
mapfile -t raw_urls < "$URLS_FILE"
fi
# Trim whitespace and remove empty entries
for u in "${raw_urls[@]:-}"; do
u="$(echo "$u" | sed 's/^[[:space:]]*//; s/[[:space:]]*$//')"
[[ -n "$u" ]] && URL_LIST+=("$u")
done
}
# Creates the playlist file for ffmpeg's concat demuxer
build_playlist() {
: > "$PLAYLIST_FILE"
for u in "${URL_LIST[@]}"; do
printf "file '%s'\n" "$u" >> "$PLAYLIST_FILE"
done
}
# Sets OUTPUT_FORMAT_ARGS based on the streaming protocol
get_output_format_args() {
case "$PROTOCOL" in
udp) OUTPUT_FORMAT_ARGS=(-f mpegts) ;;
rtp) OUTPUT_FORMAT_ARGS=(-f rtp) ;;
rtmp) OUTPUT_FORMAT_ARGS=(-f flv) ;;
icecast) OUTPUT_FORMAT_ARGS=(-content_type audio/mpeg -f mp3) ;;
*) echo "Unsupported PROTOCOL: $PROTOCOL" >&2; exit 2 ;;
esac
}
# Sets CODEC_ARGS for copying or re-encoding
get_codec_args() {
if [[ "$COPY_CODEC_WHEN_POSSIBLE" == "1" && "$PROTOCOL" != "icecast" ]]; then
CODEC_ARGS=(-c:a copy)
elif [[ "$PROTOCOL" == "icecast" ]]; then
# Icecast typically requires MP3
CODEC_ARGS=(-c:a libmp3lame -b:a "$BITRATE" -ar "$SAMPLE_RATE" -ac 2)
else
# Default re-encode
CODEC_ARGS=(-c:a "$CODEC" -b:a "$BITRATE" -ar "$SAMPLE_RATE" -ac 2)
fi
}
# Runs a single ffmpeg instance
run_ffmpeg() {
local proto_whitelist="file,crypto,data,subfile,http,https,tcp,tls,pipe"
# Run in background to allow this script to wait and manage it
ffmpeg -hide_banner -nostats -v info \
-protocol_whitelist "$proto_whitelist" \
-re -stream_loop -1 -f concat -safe 0 -i "$PLAYLIST_FILE" \
-vn "${CODEC_ARGS[@]}" \
"${OUTPUT_FORMAT_ARGS[@]}" \
$FFMPEG_EXTRA_ARGS \
"$TARGET" &
echo $! > "$PID_FILE"
wait $!
}
# --- Main Execution ---
main() {
get_urls
if [[ ${#URL_LIST[@]} -eq 0 ]]; then
echo "No URLs provided. Set URLS env or mount a file at $URLS_FILE." >&2
exit 1
fi
get_output_format_args
get_codec_args
build_playlist
echo "Starting stream from ${#URL_LIST[@]} URL(s) → $PROTOCOL$TARGET"
if [[ "$LOOP" == "1" ]]; then
while true; do
set +e # Prevent exit on non-zero ffmpeg exit code
run_ffmpeg
local exit_code=$?
set -e
echo "FFmpeg exited with code $exit_code; restarting in 2s..."
sleep 2
# Rebuild playlist in case URLs have changed/expired
build_playlist
done
else
run_ffmpeg
fi
}
main "$@"

6
s3-ffmpeg-stream/package-lock.json generated Normal file
View File

@@ -0,0 +1,6 @@
{
"name": "minio-audio-streamer",
"lockfileVersion": 3,
"requires": true,
"packages": {}
}

0
s3-nodejs-api/.bunfig Normal file
View File

View File

@@ -0,0 +1,14 @@
# This file specifies files and directories to ignore when building the Docker image.
node_modules
dist
*.log
# Keep TS sources so the build stage can compile them
# *.ts
*.map
.git
.gitignore
k8s/*
README.md
package-lock.json
.env
.DS_Store

2
s3-nodejs-api/.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
node_modules/
dist/

30
s3-nodejs-api/Dockerfile Normal file
View File

@@ -0,0 +1,30 @@
# syntax=docker/dockerfile:1
# ARG NODE_VERSION=20-alpine
# FROM node:${NODE_VERSION} AS base
FROM oven/bun:latest AS base
WORKDIR /app
FROM base AS deps
COPY package.json ./
# Install only production dependencies for the runtime image
RUN bun install --omit=dev
FROM base AS build
COPY package.json ./
# Install all dependencies for building (includes devDeps like typescript)
RUN bun install
COPY tsconfig.json ./
COPY src ./src
RUN bun run build
FROM base AS runner
ENV NODE_ENV=production
USER bun
WORKDIR /app
COPY --from=deps /app/node_modules ./node_modules
COPY package.json ./
COPY --from=build /app/dist ./dist
EXPOSE 3000
CMD ["bun", "dist/app.js"]

104
s3-nodejs-api/README.md Normal file
View File

@@ -0,0 +1,104 @@
# s3-audio-streamer
This project is designed to stream audio files in M4A format stored in AWS S3. It utilizes an Express.js server to handle streaming requests and serves audio files efficiently.
## Project Structure
- **src/**: Contains the source code for the application.
- **app.ts**: Entry point of the application, initializes the Express app and sets up middleware and routes.
- **controllers/**: Contains the StreamController which handles streaming requests.
- **routes/**: Defines the routes for the application, including streaming routes.
- **services/**: Contains the S3Client for interacting with AWS S3 and the StreamService for processing streaming requests.
- **middlewares/**: Includes middleware for error handling and request logging.
- **utils/**: Contains utility functions, such as range parsing for streaming.
- **types/**: Defines TypeScript interfaces used throughout the application.
- **k8s/**: Contains Kubernetes configuration files for deploying the application.
- **configmap.yaml**: ConfigMap for storing configuration data.
- **deployment.yaml**: Deployment configuration for the application.
- **hpa.yaml**: Horizontal Pod Autoscaler configuration.
- **ingress.yaml**: Ingress resource for routing external traffic.
- **secret.yaml**: Secret for storing sensitive information.
- **service.yaml**: Service resource for exposing the application.
## Setup Instructions
1. **Clone the repository**:
```
git clone <repository-url>
cd s3-audio-streamer
```
2. **Install dependencies**:
```
npm install
```
3. **Configure AWS credentials**:
Ensure that your AWS credentials are set up in the environment or in a configuration file.
4. **Build the Docker image**:
```
docker build -t s3-audio-streamer .
```
5. **Deploy to Kubernetes**:
Apply the Kubernetes configurations:
```
kubectl apply -f k8s/
```
## Configuration
Environment variables used by the service:
- S3_BUCKET: Name of the bucket that stores your audio files (required).
- AWS_REGION: AWS region (e.g., us-east-1). Required for AWS S3.
- S3_ENDPOINT: Optional custom S3-compatible endpoint (e.g., http://minio.minio.svc.cluster.local:9000) for MinIO/self-hosted.
- S3_FORCE_PATH_STYLE: "true" for MinIO; for AWS S3 use "false" or omit.
- S3_PREFIX: Optional key prefix/folder within the bucket (e.g., current). If set, the service will request `S3_PREFIX/<fileName>`.
- AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY: Credentials to access S3/MinIO.
- PORT: Port the server listens on (default 3000).
Kubernetes wiring:
- ConfigMap `audio-streamer-config` provides S3_BUCKET, AWS_REGION, S3_ENDPOINT, S3_FORCE_PATH_STYLE, and PORT.
- Secret `aws-secret` provides access-key-id and secret-access-key.
- Deployment consumes both via `env` refs and exposes port 3000.
- Service `s3-audio-streamer` exposes the pod on port 3000.
- Ingress `audio-streamer-ingress` routes HTTP traffic to the Service.
Health endpoints:
- GET `/` -> 200 OK (simple health)
- GET `/healthz` -> 200 OK JSON { status: 'ok' }
Streaming endpoint:
- GET `/api/streams/:fileName` streams the M4A file with optional `Range` header for byte-range requests.
Example MinIO configuration in ConfigMap:
```
S3_BUCKET=my-audio
AWS_REGION=us-east-1
S3_ENDPOINT=http://minio.minio.svc.cluster.local:9000
S3_FORCE_PATH_STYLE=true
S3_PREFIX=current
PORT=3000
```
Remember to base64-encode values for the Secret:
```
echo -n 'minio' | base64 # access-key-id
echo -n 'minio123' | base64 # secret-access-key
```
## Usage
Once deployed, the application will be accessible via the configured Ingress. You can stream audio files by sending requests to the appropriate endpoints defined in the routes.
## License
This project is licensed under the MIT License.

1224
s3-nodejs-api/bun.lock Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,15 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: audio-streamer-config
data:
S3_BUCKET: "zik"
AWS_REGION: "us-east-1"
# If using MinIO or a custom S3 endpoint, set S3_ENDPOINT, otherwise leave blank
S3_ENDPOINT: "http://minio.minio.svc.cluster.local:9000"
# S3_ENDPOINT: "https://api-data.squareserver.net"
# For MinIO, set to "true"; for AWS S3, leave empty or "false"
S3_FORCE_PATH_STYLE: "true"
# Optional object key prefix inside the bucket (e.g., 'current')
S3_PREFIX: "current"
PORT: "3000"

View File

@@ -0,0 +1,83 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: s3-audio-streamer
labels:
app: s3-audio-streamer
spec:
replicas: 2
selector:
matchLabels:
app: s3-audio-streamer
template:
metadata:
labels:
app: s3-audio-streamer
spec:
containers:
- name: s3-audio-streamer
image: gitea-http.taildb3494.ts.net/will/s3-audio-streamer
ports:
- containerPort: 3000
env:
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: aws-secret
key: access-key-id
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: aws-secret
key: secret-access-key
- name: AWS_REGION
valueFrom:
configMapKeyRef:
name: audio-streamer-config
key: AWS_REGION
- name: S3_BUCKET
valueFrom:
configMapKeyRef:
name: audio-streamer-config
key: S3_BUCKET
- name: S3_ENDPOINT
valueFrom:
configMapKeyRef:
name: audio-streamer-config
key: S3_ENDPOINT
- name: S3_FORCE_PATH_STYLE
valueFrom:
configMapKeyRef:
name: audio-streamer-config
key: S3_FORCE_PATH_STYLE
- name: S3_PREFIX
valueFrom:
configMapKeyRef:
name: audio-streamer-config
key: S3_PREFIX
- name: PORT
valueFrom:
configMapKeyRef:
name: audio-streamer-config
key: PORT
readinessProbe:
httpGet:
path: /healthz
port: 3000
initialDelaySeconds: 3
periodSeconds: 10
livenessProbe:
httpGet:
path: /healthz
port: 3000
initialDelaySeconds: 10
periodSeconds: 20
resources:
requests:
memory: "256Mi"
cpu: "500m"
limits:
memory: "512Mi"
cpu: "1"
imagePullSecrets:
- name: gitea-reg

View File

@@ -0,0 +1,19 @@
apiVersion: autoscaling/v2beta2
kind: HorizontalPodAutoscaler
metadata:
name: s3-audio-streamer-hpa
namespace: default
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: s3-audio-streamer
minReplicas: 1
maxReplicas: 10
metrics:
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: 50

View File

@@ -0,0 +1,19 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: audio-streamer-ingress
annotations:
nginx.ingress.kubernetes.io/rewrite-target: /
spec:
rules:
- host: s3-audio-streamer.192.168.153.243.nip.io
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: s3-audio-streamer
port:
number: 3000
ingressClassName: nginx

View File

@@ -0,0 +1,9 @@
apiVersion: v1
kind: Secret
metadata:
name: aws-secret
namespace: default
type: Opaque
data:
access-key-id: bXZ3ZldTc1Q3b2RkdTE0amxuMHk=
secret-access-key: Y1dscWVsQTFabGxTU0ZsdU4xTktWRnBxU2s1MFRrbDFNVkpSZW01UlkyaHdkREpCYVRkTmRBbz0=

View File

@@ -0,0 +1,14 @@
apiVersion: v1
kind: Service
metadata:
name: s3-audio-streamer
labels:
app: s3-audio-streamer
spec:
type: ClusterIP
ports:
- port: 3000
targetPort: 3000
protocol: TCP
selector:
app: s3-audio-streamer

View File

@@ -0,0 +1,30 @@
{
"name": "s3-audio-streamer",
"version": "1.0.0",
"description": "A simple application to stream audio files stored in S3.",
"main": "dist/app.js",
"scripts": {
"start": "bun dist/app.js",
"start:dev": "ts-node src/app.ts",
"build": "tsc -p tsconfig.json",
"test": "jest"
},
"dependencies": {
"aws-sdk": "^2.1234.0",
"express": "^4.17.1",
"morgan": "^1.10.0"
},
"devDependencies": {
"@types/aws-sdk": "^2.7.0",
"@types/express": "^4.17.0",
"@types/jest": "^26.0.0",
"@types/node": "^20.11.0",
"jest": "^26.6.0",
"ts-jest": "^26.5.0",
"ts-node": "^9.1.1",
"typescript": "^4.1.3"
},
"engines": {
"node": ">=12.0.0"
}
}

18
s3-nodejs-api/src/app.ts Normal file
View File

@@ -0,0 +1,18 @@
import express from 'express';
import { setRoutes } from './routes/index';
import { errorHandler } from './middlewares/errorHandler';
import requestLogger from './middlewares/requestLogger';
const app = express();
const PORT = process.env.PORT || 3000;
app.use(requestLogger);
app.use(express.json());
setRoutes(app);
app.use(errorHandler);
app.listen(PORT, () => {
console.log(`Server is running on port ${PORT}`);
});

View File

@@ -0,0 +1,23 @@
import { Request, Response } from 'express';
import StreamService from '../services/streamService';
export default class StreamController {
constructor(private streamService: StreamService) { }
// GET /api/streams/:key
async streamAudio(req: Request, res: Response) {
const { fileName: key } = req.params as { fileName: string };
const range = req.headers.range as string | undefined;
if (!key) return res.status(400).send('Missing file key');
try {
const result = await this.streamService.streamFromS3({ key, range });
res.writeHead(result.status, result.headers);
result.body.pipe(res);
} catch (err: any) {
const message = err?.message || 'Error streaming audio';
const status = err?.statusCode || 500;
res.status(status).send(message);
}
}
}

View File

@@ -0,0 +1,10 @@
import { ErrorRequestHandler } from 'express';
export const errorHandler: ErrorRequestHandler = (err, req, res, _next) => {
const message = err?.message || 'An error occurred';
const status = (err as any)?.statusCode || 500;
if (process.env.NODE_ENV !== 'test') {
console.error(err?.stack || message);
}
res.status(status).json({ message, error: message });
};

View File

@@ -0,0 +1,8 @@
import { Request, Response, NextFunction } from 'express';
const requestLogger = (req: Request, res: Response, next: NextFunction) => {
console.log(`${req.method} ${req.url}`);
next();
};
export default requestLogger;

View File

@@ -0,0 +1,18 @@
import { Application, Router, Request, Response } from 'express';
import StreamController from '../controllers/streamController';
import StreamService from '../services/streamService';
const router = Router();
const streamService = new StreamService();
const streamController = new StreamController(streamService);
export function setRoutes(app: Application) {
// Health endpoints
app.get('/', (_req: Request, res: Response) => res.status(200).send('OK'));
app.get('/healthz', (_req: Request, res: Response) => res.status(200).json({ status: 'ok' }));
app.use('/api/streams', router);
router.get('/:fileName', streamController.streamAudio.bind(streamController));
}
export default router;

View File

@@ -0,0 +1,11 @@
import { Router } from 'express';
import StreamController from '../controllers/streamController';
import StreamService from '../services/streamService';
const router = Router();
const streamService = new StreamService();
const streamController = new StreamController(streamService);
router.get('/:fileName', streamController.streamAudio.bind(streamController));
export default router;

View File

@@ -0,0 +1,49 @@
import { S3 } from 'aws-sdk';
import { Readable } from 'stream';
export interface GetObjectStreamResult {
stream: Readable;
contentLength: number; // length of the returned body
totalLength: number; // full object size
contentType?: string;
contentRange?: string;
}
export default class S3Client {
private s3: S3;
constructor() {
this.s3 = new S3({
region: process.env.AWS_REGION,
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
s3ForcePathStyle: process.env.S3_FORCE_PATH_STYLE === 'true',
endpoint: process.env.S3_ENDPOINT || undefined,
});
}
async headObject(bucket: string, key: string) {
return this.s3.headObject({ Bucket: bucket, Key: key }).promise();
}
async getObjectStream(bucket: string, key: string, range?: { start: number; end: number }): Promise<GetObjectStreamResult> {
const head = await this.headObject(bucket, key);
const totalLength = head.ContentLength || 0;
const contentType = head.ContentType;
const params: S3.GetObjectRequest = { Bucket: bucket, Key: key };
let contentRange: string | undefined;
if (range && totalLength > 0) {
params.Range = `bytes=${range.start}-${range.end}`;
contentRange = `bytes ${range.start}-${range.end}/${totalLength}`;
}
const req = this.s3.getObject(params);
const stream = req.createReadStream();
// When a range is requested, S3 returns partial content length (end-start+1)
const contentLength = range && totalLength > 0 ? range.end - range.start + 1 : totalLength;
return { stream, contentLength, totalLength, contentType, contentRange };
}
}

View File

@@ -0,0 +1,57 @@
import { Readable } from 'stream';
import S3Client, { GetObjectStreamResult } from './s3Client';
import { rangeParser } from '../utils/rangeParser';
export interface StreamResult {
status: number;
headers: Record<string, string>;
body: Readable;
}
export default class StreamService {
private s3: S3Client;
private bucket: string;
private prefix: string;
constructor(bucket = process.env.S3_BUCKET || '', prefix = process.env.S3_PREFIX || '') {
this.s3 = new S3Client();
this.bucket = bucket;
this.prefix = (prefix || '').replace(/^\/+|\/+$/g, ''); // trim leading/trailing '/'
}
async streamFromS3({ key, range }: { key: string; range?: string }): Promise<StreamResult> {
if (!this.bucket) throw new Error('S3_BUCKET not configured');
const actualKey = this.buildKey(key);
// Determine byte range
let parsedRange: { start: number; end: number } | undefined;
// We need object size to parse range properly, so head first
const head = await this.s3.headObject(this.bucket, actualKey);
const total = head.ContentLength || 0;
if (range) {
const r = rangeParser(range, total);
if (r) parsedRange = r;
}
const { stream, contentLength, totalLength, contentType, contentRange }: GetObjectStreamResult =
await this.s3.getObjectStream(this.bucket, actualKey, parsedRange);
const isPartial = Boolean(parsedRange);
const status = isPartial ? 206 : 200;
const headers: Record<string, string> = {
'Content-Type': contentType || 'audio/mp4',
'Accept-Ranges': 'bytes',
'Content-Length': String(contentLength),
};
if (isPartial && contentRange) headers['Content-Range'] = contentRange;
return { status, headers, body: stream };
}
private buildKey(key: string): string {
const cleanKey = (key || '').replace(/^\/+/, '');
if (!this.prefix) return cleanKey;
return `${this.prefix}/${cleanKey}`;
}
}

View File

@@ -0,0 +1,17 @@
export interface AudioFile {
key: string;
bucket: string;
contentType: string;
size: number;
}
export interface StreamRequest {
range: string;
audioFile: AudioFile;
}
export interface StreamResponse {
status: number;
headers: Record<string, string>;
body: Buffer;
}

View File

@@ -0,0 +1,15 @@
export function rangeParser(range: string, size: number): { start: number; end: number } | null {
if (!range) {
return null;
}
const parts = range.replace(/bytes=/, '').split('-');
const start = parseInt(parts[0], 10);
const end = parts[1] ? parseInt(parts[1], 10) : size - 1;
if (isNaN(start) || isNaN(end) || start > end || start >= size) {
return null;
}
return { start, end };
}

View File

@@ -0,0 +1,14 @@
{
"compilerOptions": {
"target": "ES2020",
"module": "commonjs",
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"outDir": "./dist",
"rootDir": "./src"
},
"include": ["src/**/*.ts"],
"exclude": ["node_modules", "**/*.spec.ts"]
}