feat(api): migrate S3 client with structured logs
This commit is contained in:
William Valentin
@@ -1,49 +1,113 @@
|
||||
import { S3 } from 'aws-sdk';
|
||||
import { Readable } from 'stream';
|
||||
import {
|
||||
S3Client as AwsS3Client,
|
||||
HeadObjectCommand,
|
||||
GetObjectCommand,
|
||||
} from "@aws-sdk/client-s3";
|
||||
import { Readable } from "stream";
|
||||
import { Logger } from "pino";
|
||||
|
||||
export interface GetObjectStreamResult {
|
||||
stream: Readable;
|
||||
contentLength: number; // length of the returned body
|
||||
totalLength: number; // full object size
|
||||
contentType?: string;
|
||||
contentRange?: string;
|
||||
stream: Readable;
|
||||
contentLength: number; // length of the returned body
|
||||
contentType?: string;
|
||||
contentRange?: string;
|
||||
}
|
||||
|
||||
export default class S3Client {
|
||||
private s3: S3;
|
||||
private s3: AwsS3Client;
|
||||
private logger: Logger;
|
||||
|
||||
constructor() {
|
||||
this.s3 = new S3({
|
||||
region: process.env.AWS_REGION,
|
||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
|
||||
s3ForcePathStyle: process.env.S3_FORCE_PATH_STYLE === 'true',
|
||||
endpoint: process.env.S3_ENDPOINT || undefined,
|
||||
});
|
||||
constructor(logger: Logger) {
|
||||
this.logger = logger;
|
||||
|
||||
const accessKeyId = process.env.AWS_ACCESS_KEY_ID;
|
||||
const secretAccessKey = process.env.AWS_SECRET_ACCESS_KEY;
|
||||
|
||||
if (!accessKeyId || !secretAccessKey) {
|
||||
throw new Error(
|
||||
"AWS credentials not configured. Set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY.",
|
||||
);
|
||||
}
|
||||
|
||||
async headObject(bucket: string, key: string) {
|
||||
return this.s3.headObject({ Bucket: bucket, Key: key }).promise();
|
||||
const config = {
|
||||
region: process.env.AWS_REGION,
|
||||
credentials: {
|
||||
accessKeyId,
|
||||
secretAccessKey,
|
||||
},
|
||||
forcePathStyle: process.env.S3_FORCE_PATH_STYLE === "true",
|
||||
endpoint: process.env.S3_ENDPOINT || undefined,
|
||||
};
|
||||
|
||||
this.logger.info(
|
||||
{
|
||||
region: config.region,
|
||||
endpoint: config.endpoint,
|
||||
forcePathStyle: config.forcePathStyle,
|
||||
},
|
||||
"S3Client: Initializing with config",
|
||||
);
|
||||
|
||||
this.s3 = new AwsS3Client(config);
|
||||
}
|
||||
|
||||
async headObject(bucket: string, key: string) {
|
||||
this.logger.info({ bucket, key }, `S3Client: headObject`);
|
||||
try {
|
||||
const command = new HeadObjectCommand({ Bucket: bucket, Key: key });
|
||||
const result = await this.s3.send(command);
|
||||
this.logger.info(
|
||||
{ size: result.ContentLength, type: result.ContentType },
|
||||
`S3Client: headObject success`,
|
||||
);
|
||||
return result;
|
||||
} catch (error) {
|
||||
this.logger.error({ error, bucket, key }, `S3Client: headObject error`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async getObjectStream(
|
||||
bucket: string,
|
||||
key: string,
|
||||
totalLength: number,
|
||||
contentType: string | undefined,
|
||||
range?: { start: number; end: number },
|
||||
): Promise<GetObjectStreamResult> {
|
||||
this.logger.info(
|
||||
{ bucket, key, range: range ? `${range.start}-${range.end}` : "none" },
|
||||
`S3Client: getObjectStream`,
|
||||
);
|
||||
|
||||
const params = {
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
Range: undefined as string | undefined,
|
||||
};
|
||||
let contentRange: string | undefined;
|
||||
if (range && totalLength > 0) {
|
||||
params.Range = `bytes=${range.start}-${range.end}`;
|
||||
contentRange = `bytes ${range.start}-${range.end}/${totalLength}`;
|
||||
}
|
||||
|
||||
async getObjectStream(bucket: string, key: string, range?: { start: number; end: number }): Promise<GetObjectStreamResult> {
|
||||
const head = await this.headObject(bucket, key);
|
||||
const totalLength = head.ContentLength || 0;
|
||||
const contentType = head.ContentType;
|
||||
try {
|
||||
const command = new GetObjectCommand(params);
|
||||
const result = await this.s3.send(command);
|
||||
const stream = result.Body as Readable;
|
||||
|
||||
const params: S3.GetObjectRequest = { Bucket: bucket, Key: key };
|
||||
let contentRange: string | undefined;
|
||||
if (range && totalLength > 0) {
|
||||
params.Range = `bytes=${range.start}-${range.end}`;
|
||||
contentRange = `bytes ${range.start}-${range.end}/${totalLength}`;
|
||||
}
|
||||
const contentLength = result.ContentLength || 0;
|
||||
|
||||
const req = this.s3.getObject(params);
|
||||
const stream = req.createReadStream();
|
||||
|
||||
// When a range is requested, S3 returns partial content length (end-start+1)
|
||||
const contentLength = range && totalLength > 0 ? range.end - range.start + 1 : totalLength;
|
||||
|
||||
return { stream, contentLength, totalLength, contentType, contentRange };
|
||||
this.logger.info(
|
||||
{ contentLength, totalLength },
|
||||
`S3Client: getObjectStream success`,
|
||||
);
|
||||
return { stream, contentLength, contentType, contentRange };
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
{ error, bucket, key },
|
||||
`S3Client: getObjectStream error`,
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,57 +1,101 @@
|
||||
import { Readable } from 'stream';
|
||||
import S3Client, { GetObjectStreamResult } from './s3Client';
|
||||
import { rangeParser } from '../utils/rangeParser';
|
||||
import { Readable } from "stream";
|
||||
import S3Client, { GetObjectStreamResult } from "./s3Client";
|
||||
import { rangeParser } from "../utils/rangeParser";
|
||||
import { Logger } from "pino";
|
||||
|
||||
export interface StreamResult {
|
||||
status: number;
|
||||
headers: Record<string, string>;
|
||||
body: Readable;
|
||||
status: number;
|
||||
headers: Record<string, string>;
|
||||
body: Readable;
|
||||
}
|
||||
|
||||
export default class StreamService {
|
||||
private s3: S3Client;
|
||||
private bucket: string;
|
||||
private prefix: string;
|
||||
private s3: S3Client;
|
||||
private bucket: string;
|
||||
private prefix: string;
|
||||
private logger: Logger;
|
||||
|
||||
constructor(bucket = process.env.S3_BUCKET || '', prefix = process.env.S3_PREFIX || '') {
|
||||
this.s3 = new S3Client();
|
||||
this.bucket = bucket;
|
||||
this.prefix = (prefix || '').replace(/^\/+|\/+$/g, ''); // trim leading/trailing '/'
|
||||
constructor(
|
||||
logger: Logger,
|
||||
bucket = process.env.S3_BUCKET || "",
|
||||
prefix = process.env.S3_PREFIX || "",
|
||||
) {
|
||||
this.s3 = new S3Client(logger);
|
||||
this.bucket = bucket;
|
||||
this.prefix = (prefix || "").replace(/^\/+|\/+$/g, ""); // trim leading/trailing '/'
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
async streamFromS3({
|
||||
key,
|
||||
range,
|
||||
}: {
|
||||
key: string;
|
||||
range?: string;
|
||||
}): Promise<StreamResult> {
|
||||
if (!this.bucket) throw new Error("S3_BUCKET not configured");
|
||||
|
||||
const actualKey = this.buildKey(key);
|
||||
this.logger.info(
|
||||
`StreamService: Attempting to stream from bucket: ${this.bucket}, key: ${actualKey}`,
|
||||
);
|
||||
|
||||
try {
|
||||
// Determine byte range
|
||||
let parsedRange: { start: number; end: number } | undefined;
|
||||
// We need object size to parse range properly, so head first
|
||||
this.logger.info(`StreamService: Getting head object for ${actualKey}`);
|
||||
const head = await this.s3.headObject(this.bucket, actualKey);
|
||||
const total = head.ContentLength || 0;
|
||||
const contentType = head.ContentType;
|
||||
this.logger.info(
|
||||
`StreamService: Object size: ${total}, content-type: ${contentType}`,
|
||||
);
|
||||
|
||||
if (range) {
|
||||
const r = rangeParser(range, total);
|
||||
if (r) parsedRange = r;
|
||||
this.logger.info(
|
||||
`StreamService: Parsed range: ${r ? `${r.start}-${r.end}` : "invalid"}`,
|
||||
);
|
||||
}
|
||||
|
||||
this.logger.info(`StreamService: Getting object stream for ${actualKey}`);
|
||||
const { stream, contentLength, contentRange }: GetObjectStreamResult =
|
||||
await this.s3.getObjectStream(
|
||||
this.bucket,
|
||||
actualKey,
|
||||
total,
|
||||
contentType,
|
||||
parsedRange,
|
||||
);
|
||||
|
||||
const isPartial = Boolean(parsedRange);
|
||||
const status = isPartial ? 206 : 200;
|
||||
const headers: Record<string, string> = {
|
||||
"Content-Type": contentType || "audio/mp4",
|
||||
"Accept-Ranges": "bytes",
|
||||
"Content-Length": String(contentLength),
|
||||
};
|
||||
if (isPartial && contentRange) headers["Content-Range"] = contentRange;
|
||||
|
||||
this.logger.info(
|
||||
{ headers, status },
|
||||
`StreamService: Successfully prepared stream`,
|
||||
);
|
||||
return { status, headers, body: stream };
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
{ error, actualKey },
|
||||
`StreamService: Error in streamFromS3`,
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async streamFromS3({ key, range }: { key: string; range?: string }): Promise<StreamResult> {
|
||||
if (!this.bucket) throw new Error('S3_BUCKET not configured');
|
||||
|
||||
const actualKey = this.buildKey(key);
|
||||
|
||||
// Determine byte range
|
||||
let parsedRange: { start: number; end: number } | undefined;
|
||||
// We need object size to parse range properly, so head first
|
||||
const head = await this.s3.headObject(this.bucket, actualKey);
|
||||
const total = head.ContentLength || 0;
|
||||
if (range) {
|
||||
const r = rangeParser(range, total);
|
||||
if (r) parsedRange = r;
|
||||
}
|
||||
|
||||
const { stream, contentLength, totalLength, contentType, contentRange }: GetObjectStreamResult =
|
||||
await this.s3.getObjectStream(this.bucket, actualKey, parsedRange);
|
||||
|
||||
const isPartial = Boolean(parsedRange);
|
||||
const status = isPartial ? 206 : 200;
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': contentType || 'audio/mp4',
|
||||
'Accept-Ranges': 'bytes',
|
||||
'Content-Length': String(contentLength),
|
||||
};
|
||||
if (isPartial && contentRange) headers['Content-Range'] = contentRange;
|
||||
|
||||
return { status, headers, body: stream };
|
||||
}
|
||||
|
||||
private buildKey(key: string): string {
|
||||
const cleanKey = (key || '').replace(/^\/+/, '');
|
||||
if (!this.prefix) return cleanKey;
|
||||
return `${this.prefix}/${cleanKey}`;
|
||||
}
|
||||
}
|
||||
private buildKey(key: string): string {
|
||||
const cleanKey = (key || "").replace(/^\/+/, "");
|
||||
if (!this.prefix) return cleanKey;
|
||||
return `${this.prefix}/${cleanKey}`;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user