diff --git a/.docker/APKBUILD b/.docker/APKBUILD new file mode 100644 index 0000000..eb0f0b5 --- /dev/null +++ b/.docker/APKBUILD @@ -0,0 +1,33 @@ +# Based on https://github.com/alpinelinux/aports/blob/a7514b6cb4a2a50704a9685a94519a9d6307f276/community/fdk-aac/APKBUILD +# This APKBUILD uses non-stripped version of fdk-aac. +pkgname=fdk-aac-nonfree +pkgver=2.0.2 +pkgrel=4 +pkgdesc="Fraunhofer FDK AAC codec library with non-free components" +url="https://github.com/mstorsjo/fdk-aac" +arch="all" +license="FDK-AAC" +makedepends="cmake samurai" +source="https://github.com/mstorsjo/fdk-aac/archive/refs/tags/v$pkgver.zip" +options="!check" # no upstream/available testsuite +builddir="$srcdir/fdk-aac-$pkgver" + +build() { + cmake -B build -G Ninja \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DBUILD_SHARED_LIBS=True \ + -DCMAKE_BUILD_TYPE=MinSizeRel + + cmake --build build +} + +package() { + DESTDIR="$pkgdir" cmake --install build + install -Dm644 NOTICE \ + "$pkgdir"/usr/share/licenses/libfdk-aac/NOTICE +} + +sha512sums=" +003f23a6b9e14757905b9cd331a442ed001ed7b835aae2c75f93db2d3a51ec15b5c4366a21bb211195cca2254e8a9805dab33a16fa670f39bd265d00b7ba5493 v2.0.2.zip +" \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 14df900..3dc3f91 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,4 +16,7 @@ tower-http = { version = "0.5.2", features = ["trace"] } num_cpus = "1.16.0" ffmpeg-next = "7.0.1" ureq = { version = "*", features = ["json", "charset"] } -mime_guess = "2.0.4" \ No newline at end of file +http = "1.1.0" +tokio-util = { version = "0.7.11", features = ["io"] } +futures-util = "0.3.30" +infer = "0.15.0" diff --git a/Dockerfile b/Dockerfile index e4a8c6e..bf7db89 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,6 @@ FROM alpine:3.20 AS builder RUN apk add --no-cache \ rust \ cargo \ - fdk-aac \ fdk-aac-dev \ clang16 \ clang16-static \ @@ -16,16 +15,9 @@ RUN apk add --no-cache \ build-base \ nasm \ yasm \ - aom-dev \ - dav1d-dev \ lame-dev \ opus-dev \ - svt-av1-dev \ libvorbis-dev \ - libvpx-dev \ - x264-dev \ - x265-dev \ - numactl-dev \ libass-dev \ libunistring-dev \ gnutls-dev && \ @@ -44,18 +36,11 @@ RUN apk add --no-cache \ --bindir="/ffmpeg/bin" \ --enable-gpl \ --enable-gnutls \ - --enable-libaom \ --enable-libass \ --enable-libfdk-aac \ - --enable-libfreetype \ --enable-libmp3lame \ --enable-libopus \ - --enable-libsvtav1 \ - --enable-libdav1d \ --enable-libvorbis \ - --enable-libvpx \ - --enable-libx264 \ - --enable-libx265 \ --enable-nonfree && \ PATH="/ffmpeg/bin:$PATH" make -j$(nproc) && \ make install && \ @@ -68,6 +53,18 @@ RUN cd /build && \ export PKG_CONFIG_PATH="/ffmpeg/ffmpeg_build/lib/pkgconfig" && \ cargo build --verbose --release +FROM alpine:3.20 AS fdk-builder +COPY ./.docker/APKBUILD /fdk-aac/APKBUILD +RUN apk add --no-cache sudo abuild build-base cmake samurai && \ + cd /fdk-aac && \ + adduser -G abuild -g "Alpine Package Builder" -s /bin/ash -D builder && \ + echo "builder ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers && \ + echo "root ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers && \ + chown -R builder:abuild /fdk-aac && \ + chmod 777 /tmp && \ + sudo -u builder sh -c 'abuild-keygen -ani && abuild -r' && \ + find /home/builder -name 'fdk-aac*' -exec mv {} /fdk-aac.apk \; + FROM alpine:3.20 WORKDIR / RUN apk add --no-cache \ @@ -75,11 +72,12 @@ RUN apk add --no-cache \ ffmpeg-libavformat \ ffmpeg-libavfilter \ ffmpeg-libavdevice \ - fdk-aac \ dumb-init \ mailcap \ tzdata \ gnutls +COPY --from=fdk-builder /fdk-aac.apk /tmp/fdk-aac.apk +RUN apk add --allow-untrusted /tmp/fdk-aac.apk && rm /tmp/fdk-aac.apk COPY --from=builder /build/target/release/atranscoder-rpc /usr/local/bin EXPOSE 8090 ENTRYPOINT ["/usr/bin/dumb-init", "--", "/usr/local/bin/atranscoder-rpc"] diff --git a/README.md b/README.md index f52c661..edca8b8 100644 --- a/README.md +++ b/README.md @@ -12,30 +12,36 @@ Transcoding can be done like this: 2. Upload file for transcoding: ```bash curl --location 'http://localhost:8090/enqueue' \ - --form 'file=@"/home/user/Music/test.mp3"' \ - --form 'format="adts"' \ - --form 'codec="aac"' \ - --form 'bitRate="64000"' \ - --form 'maxBitRate="64000"' \ - --form 'sampleRate="8000"' \ - --form 'channelLayout="mono"' \ - --form 'uploadUrl="http://127.0.0.1:8909/upload"' +--form 'file=@"/home/user/Music/test.mp3"' \ +--form 'format="mp4"' \ +--form 'codec="libfdk_aac"' \ +--form 'codecOpts="profile=aac_he"' \ +--form 'bitRate="160000"' \ +--form 'maxBitRate="160000"' \ +--form 'sampleRate="44100"' \ +--form 'channelLayout="stereo"' \ +--form 'callbackUrl="http://127.0.0.1:8909/callback"' +``` +3. Your `callbackUrl` will receive JSON response with job ID and error in case of failure. Error will be null if transcoding was successful. +4. You can download transcoded file like this (replace `job_id` with the ID you've received): +```bash +curl -L http://localhost:8090/get/job_id -o file.mp4 ``` -3. Your `uploadUrl` will receive JSON response with job ID and error in case of failure and the entire transcoded file contents in case of success (success request will have `X-Task-Id` header with task ID). Use `Content-Type` header to differentiate between the two data types. You can change configuration using this environment variables: - `LISTEN` - change this environment variable to change TCP listen address. Default is `0.0.0.0:8090`. - `NUM_WORKERS` - can be used to change how many threads will be used to transcode incoming files. Default is equal to logical CPUs. - `TEMP_DIR` - this can be used to change which directory should be used to store incoming downloads and transcoding results. Useful if you want to use a Docker volume for this. Default is system temp directory (`/tmp` for Linux). - `LOG_LEVEL` - changes log verbosity, default is `info`. +- `MAX_BODY_SIZE` - changes max body size for `/enqueue`. Default is 100MB. +- `FFMPEG_VERBOSE` - if set to `1` changes FFmpeg log level from quiet to trace. # Roadmap - [x] Implement somewhat acceptable error handling. - [x] Remove old conversion results and input files that are older than 1 hour. - [x] Remove input file after transcoding it. -- [x] Implement file upload to `uploadUrl` (if `Content-Type: application/json` then conversion was not successful and body contains an error info). -- [x] Remove transcoding result after uploading it to the `uploadUrl`. -- [ ] Docker image for `amd64` and `arm64` (currently only `amd64` is supported). -- [ ] ~~Restart threads in case of panic.~~ It's better to not panic. Current error handling seems ok for now. -- [ ] ~~Statically linked binary for Docker image & result docker image based on `scratch` (reduce image size).~~ Not yet, see [Dockerfile.scratch](Dockerfile.scratch). +- [x] Do not upload files directly, add download route with streaming instead. +- [ ] If FFmpeg fails, `send_error` won't be called - fix that. +- [ ] Default errors are returned in plain text. Change it to the JSON. +- [ ] Docker image for `amd64` and `arm64` (currently only `amd64` is supported because `arm64` cross-compilation with QEMU is sloooooooooooowwwww...). - [ ] Tests! \ No newline at end of file diff --git a/src/dto.rs b/src/dto.rs index 3ba6962..2e4169f 100644 --- a/src/dto.rs +++ b/src/dto.rs @@ -18,7 +18,7 @@ pub struct ConvertRequest { pub max_bit_rate: usize, pub sample_rate: i32, pub channel_layout: String, - pub upload_url: String, + pub callback_url: String, #[form_data(limit = "25MiB")] pub file: FieldData, diff --git a/src/server.rs b/src/server.rs index d4fb173..8168723 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1,11 +1,11 @@ +use std::env; use std::ffi::OsStr; -use std::path::Path; -use std::sync::Arc; use std::time::{Duration, SystemTime}; -use axum::extract::{DefaultBodyLimit, State}; +use axum::extract::{DefaultBodyLimit, Path, State}; use axum::http::StatusCode; -use axum::routing::post; +use axum::response::IntoResponse; +use axum::routing::{get, post}; use axum::{Json, Router}; use axum_typed_multipart::TypedMultipart; use tokio::fs; @@ -19,11 +19,21 @@ use crate::dto::{ConvertRequest, ConvertResponse}; use crate::task::{Task, TaskParams}; use crate::thread_pool::ThreadPool; -const CONTENT_LENGTH_LIMIT: usize = 30 * 1024 * 1024; +use axum::body::Body; +use axum::body::Bytes; +use futures_util::StreamExt; +use std::path::Path as StdPath; +use std::sync::Arc; +use tokio::fs::File; +use tokio::io::AsyncReadExt; +use tokio_util::io::ReaderStream; + +const CONTENT_LENGTH_LIMIT: usize = 100 * 1024 * 1024; const WORK_DIR_IN_OUT_LIFETIME: u64 = 60 * 60; pub struct Server { thread_pool: Arc, + max_body_size: usize, work_dir: String, } @@ -31,6 +41,9 @@ impl Server { pub(crate) fn new(thread_pool: ThreadPool, work_dir: String) -> Server { Server { thread_pool: Arc::new(thread_pool), + max_body_size: env::var("MAX_BODY_SIZE").map_or(CONTENT_LENGTH_LIMIT, |val| { + val.parse().map_or(CONTENT_LENGTH_LIMIT, |val| val) + }), work_dir, } } @@ -55,8 +68,9 @@ impl Server { let app = Router::new() .route( "/enqueue", - post(enqueue_file).layer(DefaultBodyLimit::max(CONTENT_LENGTH_LIMIT)), + post(enqueue_file).layer(DefaultBodyLimit::max(this.max_body_size)), ) + .route("/get/:identifier", get(download_file)) .with_state(this) .layer(TraceLayer::new_for_http()); @@ -71,8 +85,8 @@ async fn enqueue_file( TypedMultipart(req): TypedMultipart, ) -> (StatusCode, Json) { let task_id = Uuid::new_v4(); - let input = Path::new(&server.work_dir).join(format!("{}.in.atranscoder", task_id)); - let output = Path::new(&server.work_dir).join(format!("{}.out.atranscoder", task_id)); + let input = StdPath::new(&server.work_dir).join(format!("{}.in.atranscoder", task_id)); + let output = StdPath::new(&server.work_dir).join(format!("{}.out.atranscoder", task_id)); let file = req.file; @@ -95,7 +109,7 @@ async fn enqueue_file( max_bit_rate: req.max_bit_rate, sample_rate: req.sample_rate, channel_layout: req.channel_layout, - upload_url: req.upload_url, + callback_url: req.callback_url, input_path: input_path.to_string(), output_path: output_path.to_string(), }; @@ -116,6 +130,49 @@ async fn enqueue_file( } } +async fn download_file( + State(server): State>, + Path(identifier): Path, +) -> Result { + let file_name = format!("{}.out.atranscoder", identifier); + let file_path = StdPath::new(&server.work_dir).join(file_name); + + if !file_path.exists() { + return Err(StatusCode::NOT_FOUND); + } + + let mut file = match File::open(&file_path).await { + Ok(file) => file, + Err(_) => return Err(StatusCode::INTERNAL_SERVER_ERROR), + }; + + let mut buffer = [0; 512]; + let n = match file.read(&mut buffer).await { + Ok(n) if n > 0 => n, + _ => return Err(StatusCode::INTERNAL_SERVER_ERROR), + }; + + let mime_type = infer::get(&buffer[..n]).map_or("application/octet-stream".to_string(), |t| { + t.mime_type().to_string() + }); + + let file = match File::open(&file_path).await { + Ok(file) => file, + Err(_) => return Err(StatusCode::INTERNAL_SERVER_ERROR), + }; + + let stream = ReaderStream::new(file); + let body = Body::from_stream(stream.map(|result| match result { + Ok(bytes) => Ok(Bytes::from(bytes)), + Err(err) => Err(std::io::Error::new( + std::io::ErrorKind::Other, + err.to_string(), + )), + })); + + Ok(([(http::header::CONTENT_TYPE, mime_type)], body)) +} + fn error_response(msg: &str) -> (StatusCode, Json) { ( StatusCode::INTERNAL_SERVER_ERROR, diff --git a/src/task.rs b/src/task.rs index 4e486a8..870e3ce 100644 --- a/src/task.rs +++ b/src/task.rs @@ -2,10 +2,7 @@ use crate::dto::ConvertResponse; use crate::transcoder::{Transcoder, TranscoderParams}; use ffmpeg_next::channel_layout::ChannelLayout; use ffmpeg_next::{format, Dictionary}; -use mime_guess::from_path; use std::error::Error; -use std::fs::File; -use std::io::Read; use std::path::Path; use tracing::{debug, error}; use ureq::Response; @@ -28,7 +25,7 @@ impl Task { send_error( self.id, format!("Couldn't transcode: {}", err).as_str(), - &self.params.upload_url, + &self.params.callback_url, ) .ok(); return Err(err); @@ -36,26 +33,21 @@ impl Task { std::fs::remove_file(Path::new(&self.params.input_path)).ok(); - if let Err(err) = upload_file( - &self.id.to_string(), - &self.params.output_path, - &self.params.upload_url, - ) { + if let Err(err) = send_ok(self.id, &self.params.callback_url) { error!( - "couldn't upload result for job id={}, file path {}: {}", + "couldn't send result callback for job id={}, url {}: {}", &self.id.to_string(), - &self.params.output_path, + &self.params.callback_url, err ); } else { debug!( - "job id={} result was uploaded to {}", + "job id={} result was sent to callback {}", &self.id.to_string(), - &self.params.upload_url + &self.params.callback_url ); } - std::fs::remove_file(Path::new(&self.params.output_path)).ok(); Ok(()) } @@ -193,7 +185,7 @@ pub struct TaskParams { pub channel_layout: String, pub input_path: String, pub output_path: String, - pub upload_url: String, + pub callback_url: String, } fn send_error( @@ -211,41 +203,22 @@ fn send_error( if response.status() == 200 { Ok(response) } else { - Err(format!("Failed to send an error. Status: {}", response.status()).into()) + Err(format!("failed to send callback to {}. Status: {}", url, response.status()).into()) } } -fn upload_file>( - id: &str, - file_path: P, - url: &str, -) -> Result> { - let path = file_path.as_ref(); - let file_name = path - .file_name() - .ok_or("Invalid file path")? - .to_str() - .ok_or("Invalid file name")?; - - let mut file = File::open(path)?; - let mut buffer = Vec::new(); - file.read_to_end(&mut buffer)?; - - let mime_type = from_path(path).first_or_octet_stream(); - +fn send_ok(id: uuid::Uuid, url: &str) -> Result> { let response = ureq::post(url) - .set("Content-Type", mime_type.as_ref()) - .set( - "Content-Disposition", - &format!("attachment; filename=\"{}\"", file_name), - ) - .set("X-Task-Id", id) - .send_bytes(&buffer)?; + .set("Content-Type", "application/json") + .send_json(ConvertResponse { + id: Some(id.to_string()), + error: None, + })?; if response.status() == 200 { Ok(response) } else { - Err(format!("Failed to upload file. Status: {}", response.status()).into()) + Err(format!("failed to send callback to {}. Status: {}", url, response.status()).into()) } } diff --git a/src/transcoder.rs b/src/transcoder.rs index 8d819d0..6a5348e 100644 --- a/src/transcoder.rs +++ b/src/transcoder.rs @@ -1,14 +1,11 @@ extern crate ffmpeg_next as ffmpeg; -use std::any::Any; use std::error::Error; -use crate::task::params_to_avdictionary; use ffmpeg::{codec, filter, format, frame, media}; -use ffmpeg_next::codec::Parameters; use ffmpeg_next::error::EAGAIN; -use ffmpeg_next::Dictionary; -use tracing::log::debug; + +use crate::task::params_to_avdictionary; pub struct Transcoder { pub(crate) stream: usize,