Neur0toxine/atranscoder-rpc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

libfdk_aac with HE-AAC profiles

Browse Source
This commit is contained in:
Pavel 2024-05-27 18:36:36 +03:00
parent 6adf7401a5
commit 46c84fe381
8 changed files with 155 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
.docker/APKBUILD Normal file
View File

@ -0,0 +1,33 @@
# Based on https://github.com/alpinelinux/aports/blob/a7514b6cb4a2a50704a9685a94519a9d6307f276/community/fdk-aac/APKBUILD
# This APKBUILD uses non-stripped version of fdk-aac.
pkgname=fdk-aac-nonfree
pkgver=2.0.2
pkgrel=4
pkgdesc="Fraunhofer FDK AAC codec library with non-free components"
url="https://github.com/mstorsjo/fdk-aac"
arch="all"
license="FDK-AAC"
makedepends="cmake samurai"
source="https://github.com/mstorsjo/fdk-aac/archive/refs/tags/v$pkgver.zip"
options="!check" # no upstream/available testsuite
builddir="$srcdir/fdk-aac-$pkgver"
build() {
cmake -B build -G Ninja \
-DCMAKE_INSTALL_PREFIX=/usr \
-DCMAKE_INSTALL_LIBDIR=lib \
-DBUILD_SHARED_LIBS=True \
-DCMAKE_BUILD_TYPE=MinSizeRel
cmake --build build
}
package() {
DESTDIR="$pkgdir" cmake --install build
install -Dm644 NOTICE \
"$pkgdir"/usr/share/licenses/libfdk-aac/NOTICE
}
sha512sums="
003f23a6b9e14757905b9cd331a442ed001ed7b835aae2c75f93db2d3a51ec15b5c4366a21bb211195cca2254e8a9805dab33a16fa670f39bd265d00b7ba5493 v2.0.2.zip
"

5
Cargo.toml
View File

@ -16,4 +16,7 @@ tower-http = { version = "0.5.2", features = ["trace"] }
num_cpus = "1.16.0"
ffmpeg-next = "7.0.1"
ureq = { version = "*", features = ["json", "charset"] }
mime_guess = "2.0.4"
http = "1.1.0"
tokio-util = { version = "0.7.11", features = ["io"] }
futures-util = "0.3.30"
infer = "0.15.0"

30
Dockerfile
View File

@ -4,7 +4,6 @@ FROM alpine:3.20 AS builder
RUN apk add --no-cache \
rust \
cargo \
fdk-aac \
fdk-aac-dev \
clang16 \
clang16-static \
@ -16,16 +15,9 @@ RUN apk add --no-cache \
build-base \
nasm \
yasm \
aom-dev \
dav1d-dev \
lame-dev \
opus-dev \
svt-av1-dev \
libvorbis-dev \
libvpx-dev \
x264-dev \
x265-dev \
numactl-dev \
libass-dev \
libunistring-dev \
gnutls-dev && \
@ -44,18 +36,11 @@ RUN apk add --no-cache \
--bindir="/ffmpeg/bin" \
--enable-gpl \
--enable-gnutls \
--enable-libaom \
--enable-libass \
--enable-libfdk-aac \
--enable-libfreetype \
--enable-libmp3lame \
--enable-libopus \
--enable-libsvtav1 \
--enable-libdav1d \
--enable-libvorbis \
--enable-libvpx \
--enable-libx264 \
--enable-libx265 \
--enable-nonfree && \
PATH="/ffmpeg/bin:$PATH" make -j$(nproc) && \
make install && \
@ -68,6 +53,18 @@ RUN cd /build && \
export PKG_CONFIG_PATH="/ffmpeg/ffmpeg_build/lib/pkgconfig" && \
cargo build --verbose --release
FROM alpine:3.20 AS fdk-builder
COPY ./.docker/APKBUILD /fdk-aac/APKBUILD
RUN apk add --no-cache sudo abuild build-base cmake samurai && \
cd /fdk-aac && \
adduser -G abuild -g "Alpine Package Builder" -s /bin/ash -D builder && \
echo "builder ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers && \
echo "root ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers && \
chown -R builder:abuild /fdk-aac && \
chmod 777 /tmp && \
sudo -u builder sh -c 'abuild-keygen -ani && abuild -r' && \
find /home/builder -name 'fdk-aac*' -exec mv {} /fdk-aac.apk \;
FROM alpine:3.20
WORKDIR /
RUN apk add --no-cache \
@ -75,11 +72,12 @@ RUN apk add --no-cache \
ffmpeg-libavformat \
ffmpeg-libavfilter \
ffmpeg-libavdevice \
fdk-aac \
dumb-init \
mailcap \
tzdata \
gnutls
COPY --from=fdk-builder /fdk-aac.apk /tmp/fdk-aac.apk
RUN apk add --allow-untrusted /tmp/fdk-aac.apk && rm /tmp/fdk-aac.apk
COPY --from=builder /build/target/release/atranscoder-rpc /usr/local/bin
EXPOSE 8090
ENTRYPOINT ["/usr/bin/dumb-init", "--", "/usr/local/bin/atranscoder-rpc"]

34
README.md
View File

@ -12,30 +12,36 @@ Transcoding can be done like this:
2. Upload file for transcoding:
```bash
curl --location 'http://localhost:8090/enqueue' \
--form 'file=@"/home/user/Music/test.mp3"' \
--form 'format="adts"' \
--form 'codec="aac"' \
--form 'bitRate="64000"' \
--form 'maxBitRate="64000"' \
--form 'sampleRate="8000"' \
--form 'channelLayout="mono"' \
--form 'uploadUrl="http://127.0.0.1:8909/upload"'
--form 'file=@"/home/user/Music/test.mp3"' \
--form 'format="mp4"' \
--form 'codec="libfdk_aac"' \
--form 'codecOpts="profile=aac_he"' \
--form 'bitRate="160000"' \
--form 'maxBitRate="160000"' \
--form 'sampleRate="44100"' \
--form 'channelLayout="stereo"' \
--form 'callbackUrl="http://127.0.0.1:8909/callback"'
```
3. Your `callbackUrl` will receive JSON response with job ID and error in case of failure. Error will be null if transcoding was successful.
4. You can download transcoded file like this (replace `job_id` with the ID you've received):
```bash
curl -L http://localhost:8090/get/job_id -o file.mp4
```
3. Your `uploadUrl` will receive JSON response with job ID and error in case of failure and the entire transcoded file contents in case of success (success request will have `X-Task-Id` header with task ID). Use `Content-Type` header to differentiate between the two data types.
You can change configuration using this environment variables:
- `LISTEN` - change this environment variable to change TCP listen address. Default is `0.0.0.0:8090`.
- `NUM_WORKERS` - can be used to change how many threads will be used to transcode incoming files. Default is equal to logical CPUs.
- `TEMP_DIR` - this can be used to change which directory should be used to store incoming downloads and transcoding results. Useful if you want to use a Docker volume for this. Default is system temp directory (`/tmp` for Linux).
- `LOG_LEVEL` - changes log verbosity, default is `info`.
- `MAX_BODY_SIZE` - changes max body size for `/enqueue`. Default is 100MB.
- `FFMPEG_VERBOSE` - if set to `1` changes FFmpeg log level from quiet to trace.
# Roadmap
- [x] Implement somewhat acceptable error handling.
- [x] Remove old conversion results and input files that are older than 1 hour.
- [x] Remove input file after transcoding it.
- [x] Implement file upload to `uploadUrl` (if `Content-Type: application/json` then conversion was not successful and body contains an error info).
- [x] Remove transcoding result after uploading it to the `uploadUrl`.
- [ ] Docker image for `amd64` and `arm64` (currently only `amd64` is supported).
- [ ] ~~Restart threads in case of panic.~~ It's better to not panic. Current error handling seems ok for now.
- [ ] ~~Statically linked binary for Docker image & result docker image based on `scratch` (reduce image size).~~ Not yet, see [Dockerfile.scratch](Dockerfile.scratch).
- [x] Do not upload files directly, add download route with streaming instead.
- [ ] If FFmpeg fails, `send_error` won't be called - fix that.
- [ ] Default errors are returned in plain text. Change it to the JSON.
- [ ] Docker image for `amd64` and `arm64` (currently only `amd64` is supported because `arm64` cross-compilation with QEMU is sloooooooooooowwwww...).
- [ ] Tests!

2
src/dto.rs
View File

@ -18,7 +18,7 @@ pub struct ConvertRequest {
pub max_bit_rate: usize,
pub sample_rate: i32,
pub channel_layout: String,
pub upload_url: String,
pub callback_url: String,
#[form_data(limit = "25MiB")]
pub file: FieldData<NamedTempFile>,

75
src/server.rs
View File

@ -1,11 +1,11 @@
use std::env;
use std::ffi::OsStr;
use std::path::Path;
use std::sync::Arc;
use std::time::{Duration, SystemTime};
use axum::extract::{DefaultBodyLimit, State};
use axum::extract::{DefaultBodyLimit, Path, State};
use axum::http::StatusCode;
use axum::routing::post;
use axum::response::IntoResponse;
use axum::routing::{get, post};
use axum::{Json, Router};
use axum_typed_multipart::TypedMultipart;
use tokio::fs;
@ -19,11 +19,21 @@ use crate::dto::{ConvertRequest, ConvertResponse};
use crate::task::{Task, TaskParams};
use crate::thread_pool::ThreadPool;
const CONTENT_LENGTH_LIMIT: usize = 30 * 1024 * 1024;
use axum::body::Body;
use axum::body::Bytes;
use futures_util::StreamExt;
use std::path::Path as StdPath;
use std::sync::Arc;
use tokio::fs::File;
use tokio::io::AsyncReadExt;
use tokio_util::io::ReaderStream;
const CONTENT_LENGTH_LIMIT: usize = 100 * 1024 * 1024;
const WORK_DIR_IN_OUT_LIFETIME: u64 = 60 * 60;
pub struct Server {
thread_pool: Arc<ThreadPool>,
max_body_size: usize,
work_dir: String,
}
@ -31,6 +41,9 @@ impl Server {
pub(crate) fn new(thread_pool: ThreadPool, work_dir: String) -> Server {
Server {
thread_pool: Arc::new(thread_pool),
max_body_size: env::var("MAX_BODY_SIZE").map_or(CONTENT_LENGTH_LIMIT, |val| {
val.parse().map_or(CONTENT_LENGTH_LIMIT, |val| val)
}),
work_dir,
}
}
@ -55,8 +68,9 @@ impl Server {
let app = Router::new()
.route(
"/enqueue",
post(enqueue_file).layer(DefaultBodyLimit::max(CONTENT_LENGTH_LIMIT)),
post(enqueue_file).layer(DefaultBodyLimit::max(this.max_body_size)),
)
.route("/get/:identifier", get(download_file))
.with_state(this)
.layer(TraceLayer::new_for_http());
@ -71,8 +85,8 @@ async fn enqueue_file(
TypedMultipart(req): TypedMultipart<ConvertRequest>,
) -> (StatusCode, Json<ConvertResponse>) {
let task_id = Uuid::new_v4();
let input = Path::new(&server.work_dir).join(format!("{}.in.atranscoder", task_id));
let output = Path::new(&server.work_dir).join(format!("{}.out.atranscoder", task_id));
let input = StdPath::new(&server.work_dir).join(format!("{}.in.atranscoder", task_id));
let output = StdPath::new(&server.work_dir).join(format!("{}.out.atranscoder", task_id));
let file = req.file;
@ -95,7 +109,7 @@ async fn enqueue_file(
max_bit_rate: req.max_bit_rate,
sample_rate: req.sample_rate,
channel_layout: req.channel_layout,
upload_url: req.upload_url,
callback_url: req.callback_url,
input_path: input_path.to_string(),
output_path: output_path.to_string(),
};
@ -116,6 +130,49 @@ async fn enqueue_file(
}
}
async fn download_file(
State(server): State<Arc<Server>>,
Path(identifier): Path<String>,
) -> Result<impl IntoResponse, StatusCode> {
let file_name = format!("{}.out.atranscoder", identifier);
let file_path = StdPath::new(&server.work_dir).join(file_name);
if !file_path.exists() {
return Err(StatusCode::NOT_FOUND);
}
let mut file = match File::open(&file_path).await {
Ok(file) => file,
Err(_) => return Err(StatusCode::INTERNAL_SERVER_ERROR),
};
let mut buffer = [0; 512];
let n = match file.read(&mut buffer).await {
Ok(n) if n > 0 => n,
_ => return Err(StatusCode::INTERNAL_SERVER_ERROR),
};
let mime_type = infer::get(&buffer[..n]).map_or("application/octet-stream".to_string(), |t| {
t.mime_type().to_string()
});
let file = match File::open(&file_path).await {
Ok(file) => file,
Err(_) => return Err(StatusCode::INTERNAL_SERVER_ERROR),
};
let stream = ReaderStream::new(file);
let body = Body::from_stream(stream.map(|result| match result {
Ok(bytes) => Ok(Bytes::from(bytes)),
Err(err) => Err(std::io::Error::new(
std::io::ErrorKind::Other,
err.to_string(),
)),
}));
Ok(([(http::header::CONTENT_TYPE, mime_type)], body))
}
fn error_response(msg: &str) -> (StatusCode, Json<ConvertResponse>) {
(
StatusCode::INTERNAL_SERVER_ERROR,

57
src/task.rs
View File

@ -2,10 +2,7 @@ use crate::dto::ConvertResponse;
use crate::transcoder::{Transcoder, TranscoderParams};
use ffmpeg_next::channel_layout::ChannelLayout;
use ffmpeg_next::{format, Dictionary};
use mime_guess::from_path;
use std::error::Error;
use std::fs::File;
use std::io::Read;
use std::path::Path;
use tracing::{debug, error};
use ureq::Response;
@ -28,7 +25,7 @@ impl Task {
send_error(
self.id,
format!("Couldn't transcode: {}", err).as_str(),
&self.params.upload_url,
&self.params.callback_url,
)
.ok();
return Err(err);
@ -36,26 +33,21 @@ impl Task {
std::fs::remove_file(Path::new(&self.params.input_path)).ok();
if let Err(err) = upload_file(
&self.id.to_string(),
&self.params.output_path,
&self.params.upload_url,
) {
if let Err(err) = send_ok(self.id, &self.params.callback_url) {
error!(
"couldn't upload result for job id={}, file path {}: {}",
"couldn't send result callback for job id={}, url {}: {}",
&self.id.to_string(),
&self.params.output_path,
&self.params.callback_url,
err
);
} else {
debug!(
"job id={} result was uploaded to {}",
"job id={} result was sent to callback {}",
&self.id.to_string(),
&self.params.upload_url
&self.params.callback_url
);
}
std::fs::remove_file(Path::new(&self.params.output_path)).ok();
Ok(())
}
@ -193,7 +185,7 @@ pub struct TaskParams {
pub channel_layout: String,
pub input_path: String,
pub output_path: String,
pub upload_url: String,
pub callback_url: String,
}
fn send_error(
@ -211,41 +203,22 @@ fn send_error(
if response.status() == 200 {
Ok(response)
} else {
Err(format!("Failed to send an error. Status: {}", response.status()).into())
Err(format!("failed to send callback to {}. Status: {}", url, response.status()).into())
}
}
fn upload_file<P: AsRef<Path>>(
id: &str,
file_path: P,
url: &str,
) -> Result<Response, Box<dyn std::error::Error>> {
let path = file_path.as_ref();
let file_name = path
.file_name()
.ok_or("Invalid file path")?
.to_str()
.ok_or("Invalid file name")?;
let mut file = File::open(path)?;
let mut buffer = Vec::new();
file.read_to_end(&mut buffer)?;
let mime_type = from_path(path).first_or_octet_stream();
fn send_ok(id: uuid::Uuid, url: &str) -> Result<Response, Box<dyn std::error::Error>> {
let response = ureq::post(url)
.set("Content-Type", mime_type.as_ref())
.set(
"Content-Disposition",
&format!("attachment; filename=\"{}\"", file_name),
)
.set("X-Task-Id", id)
.send_bytes(&buffer)?;
.set("Content-Type", "application/json")
.send_json(ConvertResponse {
id: Some(id.to_string()),
error: None,
})?;
if response.status() == 200 {
Ok(response)
} else {
Err(format!("Failed to upload file. Status: {}", response.status()).into())
Err(format!("failed to send callback to {}. Status: {}", url, response.status()).into())
}
}

7
src/transcoder.rs
View File

@ -1,14 +1,11 @@
extern crate ffmpeg_next as ffmpeg;
use std::any::Any;
use std::error::Error;
use crate::task::params_to_avdictionary;
use ffmpeg::{codec, filter, format, frame, media};
use ffmpeg_next::codec::Parameters;
use ffmpeg_next::error::EAGAIN;
use ffmpeg_next::Dictionary;
use tracing::log::debug;
use crate::task::params_to_avdictionary;
pub struct Transcoder {
pub(crate) stream: usize,