xing/amlnn-model-playground
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Upload first version

Browse source
This commit is contained in:
dengliu1105 2026-01-06 10:29:54 +08:00
parent f95d5a63b0
commit 3bdf2003ec
898 changed files with 1405811 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

0
examples/whisper/cpp/.gitkeep Normal file
View file

76
examples/whisper/cpp/build-android.sh Executable file
View file

@ -0,0 +1,76 @@
#!/bin/bash
set -e
#
# Copyright (C) 2024–2025 Amlogic, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
usage() {
echo "Usage: $0 [-a <target_abi>]"
echo " -a <target_abi> : Target ABI (default: arm64-v8a)"
echo " -h : Show this help message"
exit 1
}
# Default values
TARGET_ABI=arm64-v8a
# Parse arguments
while getopts 'a:h' opt; do
case "$opt" in
a)
TARGET_ABI=$OPTARG
;;
h)
usage
;;
*)
usage
;;
esac
done
if [ -z "${ANDROID_NDK_PATH}" ]; then
if [ -n "${ANDROID_NDK}" ]; then
ANDROID_NDK_PATH=${ANDROID_NDK}
elif [ -n "${ANDROID_NDK_HOME}" ]; then
ANDROID_NDK_PATH=${ANDROID_NDK_HOME}
else
echo "Error: ANDROID_NDK_PATH is not set."
echo "Please set ANDROID_NDK_PATH to your Android NDK directory."
exit 1
fi
fi
ROOT_PWD=$(cd "$(dirname $0)" && pwd)
BUILD_DIR=${ROOT_PWD}/build/android
echo "Building for Android..."
echo "NDK_PATH: ${ANDROID_NDK_PATH}"
echo "TARGET_ABI: ${TARGET_ABI}"
echo "BUILD_DIR: ${BUILD_DIR}"
mkdir -p ${BUILD_DIR}
cd ${BUILD_DIR}
cmake ../../src \
-DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK_PATH}/build/cmake/android.toolchain.cmake \
-DANDROID_ABI=${TARGET_ABI} \
-DANDROID_PLATFORM=android-24 \
-DCMAKE_BUILD_TYPE=Release
make -j4
echo "Build complete. Executable in ${BUILD_DIR}/whisper_demo"

19
examples/whisper/cpp/build-linux.sh Executable file
View file

@ -0,0 +1,19 @@
#!/bin/bash
set -e
#
# Copyright (C) 2024–2025 Amlogic, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
### TO DO

BIN
examples/whisper/cpp/data_bin/data.bin Executable file
View file

Binary file not shown.

BIN
examples/whisper/cpp/data_bin/tokenizer_info.bin Executable file
View file

Binary file not shown.

40
examples/whisper/cpp/src/CMakeLists.txt Executable file
View file

@ -0,0 +1,40 @@
cmake_minimum_required(VERSION 3.5)
project(whisper_demo)
set(CMAKE_CXX_STANDARD 17)
# Set NNSDK path
set(NNSDK_ROOT "${CMAKE_SOURCE_DIR}/../../../../dependency/nnsdk")
include_directories(${NNSDK_ROOT}/include)
include_directories(${CMAKE_SOURCE_DIR}/../../../../common)
# Set 3rdparty path
set(3RDPARTY_DIR "${CMAKE_SOURCE_DIR}/../../../../dependency")
if(CMAKE_SYSTEM_NAME STREQUAL "Android")
if (ANDROID_ABI STREQUAL "arm64-v8a")
link_directories(${NNSDK_ROOT}/lib/android/arm64-v8a)
else()
link_directories(${NNSDK_ROOT}/lib/android/armeabi-v7a)
endif()
# Android needs log
link_libraries(log)
elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux")
link_directories(${NNSDK_ROOT}/lib/linux/lib64_yocto)
endif()
add_executable(${PROJECT_NAME}
main.cpp
common.cpp
whisper.cpp
whisper_invoke.cpp
pre_process_whisper.cpp
post_process_whisper.cpp
)
target_link_libraries(${PROJECT_NAME}
nnsdk
dl
m
)

135
examples/whisper/cpp/src/common.cpp Executable file
View file

@ -0,0 +1,135 @@
/*
* Copyright (C) 2024–2025 Amlogic, Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "common.h"
// third-party utilities
// use your favorite implementations
#define DR_WAV_IMPLEMENTATION
#include "dr_wav.h"
bool is_wav_buffer(const std::string buf) {
// RIFF ref: https://en.wikipedia.org/wiki/Resource_Interchange_File_Format
// WAV ref: https://www.mmsp.ece.mcgill.ca/Documents/AudioFormats/WAVE/WAVE.html
if (buf.size() < 12 || buf.substr(0, 4) != "RIFF" || buf.substr(8, 4) != "WAVE") {
return false;
}
uint32_t chunk_size = *reinterpret_cast<const uint32_t*>(buf.data() + 4);
if (chunk_size + 8 != buf.size()) {
return false;
}
return true;
}
bool read_wav(const std::string & fname, std::vector<float>& pcmf32, std::vector<std::vector<float>>& pcmf32s, bool stereo) {
drwav wav;
std::vector<uint8_t> wav_data; // used for pipe input from stdin
if (fname == "-") {
{
#ifdef _WIN32
_setmode(_fileno(stdin), _O_BINARY);
#endif
uint8_t buf[1024];
while (true)
{
const size_t n = fread(buf, 1, sizeof(buf), stdin);
if (n == 0) {
break;
}
wav_data.insert(wav_data.end(), buf, buf + n);
}
}
if (drwav_init_memory(&wav, wav_data.data(), wav_data.size(), nullptr) == false) {
fprintf(stderr, "error: failed to open WAV file from stdin\n");
return false;
}
fprintf(stderr, "%s: read %zu bytes from stdin\n", __func__, wav_data.size());
}
else if (is_wav_buffer(fname)) {
if (drwav_init_memory(&wav, fname.c_str(), fname.size(), nullptr) == false) {
fprintf(stderr, "error: failed to open WAV file from fname buffer\n");
return false;
}
}
else if (drwav_init_file(&wav, fname.c_str(), nullptr) == false) {
fprintf(stderr, "error: failed to open '%s' as WAV file\n", fname.c_str());
return false;
}
if (wav.channels != 1 && wav.channels != 2) {
fprintf(stderr, "%s: WAV file '%s' must be mono or stereo\n", __func__, fname.c_str());
drwav_uninit(&wav);
return false;
}
if (stereo && wav.channels != 2) {
fprintf(stderr, "%s: WAV file '%s' must be stereo for diarization\n", __func__, fname.c_str());
drwav_uninit(&wav);
return false;
}
if (wav.sampleRate != COMMON_SAMPLE_RATE) {
fprintf(stderr, "%s: WAV file '%s' must be %i kHz\n", __func__, fname.c_str(), COMMON_SAMPLE_RATE/1000);
drwav_uninit(&wav);
return false;
}
if (wav.bitsPerSample != 16) {
fprintf(stderr, "%s: WAV file '%s' must be 16-bit\n", __func__, fname.c_str());
drwav_uninit(&wav);
return false;
}
const uint64_t n = wav_data.empty() ? wav.totalPCMFrameCount : wav_data.size()/(wav.channels*wav.bitsPerSample/8);
std::vector<int16_t> pcm16;
pcm16.resize(n*wav.channels);
drwav_read_pcm_frames_s16(&wav, n, pcm16.data());
drwav_uninit(&wav);
// convert to mono, float
pcmf32.resize(n);
if (wav.channels == 1) {
for (uint64_t i = 0; i < n; i++) {
pcmf32[i] = float(pcm16[i])/32768.0f;
}
} else {
for (uint64_t i = 0; i < n; i++) {
pcmf32[i] = float(pcm16[2*i] + pcm16[2*i + 1])/65536.0f;
}
}
if (stereo) {
// convert to stereo, float
pcmf32s.resize(2);
pcmf32s[0].resize(n);
pcmf32s[1].resize(n);
for (uint64_t i = 0; i < n; i++) {
pcmf32s[0][i] = float(pcm16[2*i])/32768.0f;
pcmf32s[1][i] = float(pcm16[2*i + 1])/32768.0f;
}
}
return true;
}

40
examples/whisper/cpp/src/common.h Executable file
View file

@ -0,0 +1,40 @@
/*
* Copyright (C) 2024–2025 Amlogic, Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <string>
#include <map>
#include <vector>
#include <random>
#include <thread>
#include <ctime>
#include <fstream>
#define COMMON_SAMPLE_RATE 16000
bool is_wav_buffer(const std::string buf);
// Read WAV audio file and store the PCM data into pcmf32
// fname can be a buffer of WAV data instead of a filename
// The sample rate of the audio must be equal to COMMON_SAMPLE_RATE
// If stereo flag is set and the audio has 2 channels, the pcmf32s will contain 2 channel PCM
bool read_wav(
const std::string & fname,
std::vector<float> & pcmf32,
std::vector<std::vector<float>> & pcmf32s,
bool stereo);

6434
examples/whisper/cpp/src/dr_wav.h Executable file
View file

File diff suppressed because it is too large Load diff

204
examples/whisper/cpp/src/main.cpp Executable file
View file

@ -0,0 +1,204 @@
/*
* Copyright (C) 2024–2025 Amlogic, Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <stdio.h>
#include <time.h>
#include <iostream>
#include "whisper_invoke.h"
#include "nn_sdk.h"
#define BILLION 1000000000
#define GET_INFERENCE_TIME (1)
#define WHISPER_DECODER_INPUTS 48
struct Get_Times
{
uint64_t init_start_time, init_end_time, init_total_time;
uint64_t preProcess_start_time, preProcess_end_time, preProcess_total_time;
uint64_t invoke_start_time, invoke_end_time, invoke_total_time; /* for whisper_decoder or llm invoke time once */
uint64_t total_time; /* for whisper or llm pipeline time */
std::vector<uint64_t> total_time_group; /* for whisper_decoder or llm invoke time everytimes */
};
static uint64_t get_time_count()
{
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return (uint64_t)((uint64_t)ts.tv_nsec + (uint64_t)ts.tv_sec * BILLION);
}
int main(int argc, char ** argv)
{
Get_Times encoder_time, decoder_time, whisper_time;
Input_Decoder decoder_inputs_data;
std::vector<float> encoder_input_data;
std::vector<float> encoder_output_data;
int64_t input_1_data[] = {50257, 50362}; /* init token, for tiny_en or base_en */
int input_1_data_size = sizeof(input_1_data) / sizeof(input_1_data[0]);
int ret = 0;
char* model_path_encoder = argv[1];
char* model_path_decoder = argv[2];
void *context_enc = NULL;
void *context_dec = NULL;
whisper_time.init_start_time = get_time_count();
context_enc = init_network_file(model_path_encoder);
context_dec = init_network_file(model_path_decoder);
whisper_time.init_end_time = get_time_count();
whisper_time.init_total_time = (whisper_time.init_end_time - whisper_time.init_start_time) / 1000000;
if (context_enc == NULL)
{
printf("init_network [context_enc] fail.\n");
return -1;
}
if (context_dec == NULL)
{
printf("init_network [context_dec] fail.\n");
return -1;
}
if (getenv("GET_TIME"))
{
std::cout << "init_whisper_total time : " << whisper_time.init_total_time << "ms" << std::endl;
}
while (true)
{
std::string input_str;
bool is_finish = false;
std::string out_text = "start"; /* end adla model output text init */
printf("\n");
printf("Audio Path:\n");
std::getline(std::cin, input_str);
if (input_str == "exit")
{
break;
} else if (input_str == "") {
printf("Please enter wav path\n");
continue;
} else if (input_str.size() < 4 || input_str.substr(input_str.size() - 4) != ".wav") {
std::cout << "Invalid wav path or file does not exist, please try again" << std::endl;
continue;
}
decoder_inputs_data.input_1_size = WHISPER_DECODER_INPUTS;
decoder_inputs_data.input_1= new int64_t[decoder_inputs_data.input_1_size];
std::copy(input_1_data, input_1_data + input_1_data_size, decoder_inputs_data.input_1);
// need enough data 0
std::fill(decoder_inputs_data.input_1 + input_1_data_size,
decoder_inputs_data.input_1 + decoder_inputs_data.input_1_size,
0);
whisper_time.preProcess_start_time = get_time_count();
encoder_input_data = do_pre_process(input_str);
if (!encoder_input_data.size()) /* support wav 0s */
{
is_finish = is_finish_end();
std::cout << "wav is null, please try again" << std::endl;
continue;
}
whisper_time.preProcess_end_time = get_time_count();
encoder_output_data = run_network_encoder_process(context_enc, encoder_input_data);
encoder_time.invoke_end_time = get_time_count();
decoder_inputs_data.input_0_size = encoder_output_data.size();
decoder_inputs_data.input_0 = new float[decoder_inputs_data.input_0_size];
std::copy(encoder_output_data.begin(), encoder_output_data.end(), decoder_inputs_data.input_0);
whisper_time.preProcess_total_time = (whisper_time.preProcess_end_time - whisper_time.preProcess_start_time) / 1000000;
encoder_time.invoke_total_time = (encoder_time.invoke_end_time - whisper_time.preProcess_end_time) / 1000000;
printf("\n");
printf("Audio Text:\n");
while (!is_finish)
{
decoder_time.invoke_start_time = get_time_count();
out_text = run_network_decoder(context_dec, &decoder_inputs_data);
decoder_time.invoke_end_time = get_time_count();
is_finish = is_finish_end();
decoder_time.total_time_group.push_back((decoder_time.invoke_end_time - decoder_time.invoke_start_time) / 1000000);
std::cout << out_text << std::flush;
}
printf("\n");
if (getenv("GET_OUTPUTS_SIZE"))
{
std::cout << "==================================" << std::endl;
std::cout << "WHISPER_OUTPUTS_SIZE : " << decoder_time.total_time_group.size() << std::endl;
}
if (getenv("GET_TIME"))
{
uint64_t total_time_whisper, total_time_decoder, total_time_llm;
for (int i = 0; i < decoder_time.total_time_group.size(); i++) {
std::cout << "==================================" << std::endl;
if (i < 1)
{
total_time_whisper = whisper_time.preProcess_total_time + encoder_time.invoke_total_time;
whisper_time.total_time = whisper_time.preProcess_total_time + encoder_time.invoke_total_time;
std::cout << "pre-process time : " << whisper_time.preProcess_total_time << "ms" << std::endl;
std::cout << "encoder_inference_total time : " << encoder_time.invoke_total_time << "ms" << std::endl;
}
decoder_time.invoke_total_time += decoder_time.total_time_group[i];
std::cout << "decoder inference time[" << i << "] : " << decoder_time.total_time_group[i] << "ms" << std::endl;
}
whisper_time.total_time += decoder_time.invoke_total_time;
std::cout << "model->whisper decoder avg : " << decoder_time.invoke_total_time / decoder_time.total_time_group.size() << "ms" << std::endl;
std::cout << "model->whisper total time : " << whisper_time.total_time << "ms" << std::endl;
whisper_time.total_time = decoder_time.invoke_total_time = 0;
}
encoder_time.total_time_group.clear();
if (decoder_inputs_data.input_0 != nullptr)
{
delete[] decoder_inputs_data.input_0;
decoder_inputs_data.input_0 = nullptr;
decoder_inputs_data.input_0_size = 0;
}
if (decoder_inputs_data.input_1 != nullptr)
{
delete[] decoder_inputs_data.input_1;
decoder_inputs_data.input_1 = nullptr;
decoder_inputs_data.input_1_size = 0;
}
}
ret = destroy_network(context_enc);
if (ret != 0)
{
printf("destroy_network [context_enc] fail.\n");
return -1;
}
ret = destroy_network(context_dec);
if (ret != 0)
{
printf("destroy_network [context_dec] fail.\n");
return -1;
}
return ret;
}

127
examples/whisper/cpp/src/post_process_whisper.cpp Executable file
View file

@ -0,0 +1,127 @@
/*
* Copyright (C) 2024–2025 Amlogic, Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "pre_post_common.h"
#include "post_process_whisper.h"
whisper_vocab read_token_info(std::string token_path)
{
struct whisper_context ctx;
auto & vocab = ctx.vocab;
whisper_model_loader loader = {};
auto fin = std::ifstream(token_path, std::ios::binary);
if (!fin)
{
fprintf(stderr, "%s : fail to open '%s'\n", __func__, token_path.c_str());
}
loader.context = &fin;
loader.read = [](void * ctx, void * output, size_t read_size) {
std::ifstream * fin = (std::ifstream*)ctx;
fin->read((char *)output, read_size);
return read_size;
};
loader.eof = [](void * ctx) {
std::ifstream * fin = (std::ifstream*)ctx;
return fin->eof();
};
loader.close = [](void * ctx) {
std::ifstream * fin = (std::ifstream*)ctx;
fin->close();
};
int32_t n_vocab = 0;
read_safe(&loader, n_vocab);
std::string word;
std::vector<char> tmp;
tmp.reserve(128);
for (int i = 0; i < n_vocab; i++) {
uint32_t len;
read_safe(&loader, len);
if (len > 0 and i != 50256) {
tmp.resize(len);
loader.read(loader.context, &tmp[0], tmp.size()); // read to buffer
word.assign(tmp.data(), tmp.size());
} else {
word = "";
}
vocab.token_to_id[word] = i;
vocab.id_to_token[i] = word;
}
fin.eof();
fin.close();
n_vocab = 50256;
if (n_vocab < 51863) {
// WHISPER_LOG_INFO("%s: adding %d extra tokens\n", __func__, model.hparams.n_vocab - n_vocab);
for (int i = n_vocab; i < 51863; i++) {
if (i > vocab.token_beg) {
word = "[_TT_" + std::to_string(i - vocab.token_beg) + "]";
} else if (i == vocab.token_eot) {
word = "<|endoftext|>";
} else if (i == vocab.token_sot) {
word = "<|startoftranscript|>";
} else if (i == vocab.token_translate) {
word = "<|translate|>";
} else if (i == vocab.token_transcribe) {
word = "<|transcribe|>";
} else if (i == vocab.token_solm) {
word = "[_SOLM_]";
} else if (i == vocab.token_prev) {
word = "[_PREV_]";
} else if (i == vocab.token_nosp) {
word = "[_NOSP_]";
} else if (i == vocab.token_not) {
word = "<|notimestamps|>";
} else if (i == vocab.token_beg) {
word = "[_BEG_]";
}
else if (i == 50258) {
word= "<|en|>";
}
else if (i == 50259) {
word= "<|zh|>";
}
else if (i == 50263) {
word= "<|ko|>";
}
else {
word = "[_extra_token_" + std::to_string(i) + "]";
}
vocab.token_to_id[word] = i;
vocab.id_to_token[i] = word;
}
}
return vocab;
}
std::string do_post_process(int64_t output_id, whisper_vocab vocab)
{
// std::vector<whisper_token> prompt_init = {50258, 50259, 50359, 50363,2221,13,2326,388,391,307,264,50244,295,264,2808,5359,11,293,321,366,5404,281,2928,702,14943,13,50257};
std::string text;
text = vocab.id_to_token.at(output_id).c_str();
return text;
}

21
examples/whisper/cpp/src/post_process_whisper.h Executable file
View file

@ -0,0 +1,21 @@
/*
* Copyright (C) 2024–2025 Amlogic, Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "pre_post_common.h"
whisper_vocab read_token_info(std::string token_path);
int get_output_max_index(size_t id_shape, std::vector<float> buf_data);
std::string do_post_process(int64_t output_id, whisper_vocab vocab);

105
examples/whisper/cpp/src/pre_post_common.h Executable file
View file

@ -0,0 +1,105 @@
/*
* Copyright (C) 2024–2025 Amlogic, Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef PRE_POST_COMMON_H
#define PRE_POST_COMMON_H
#include "common.h"
#include "whisper.h"
#include <cmath>
#include <fstream>
#include <cstdio>
#include <regex>
#include <thread>
#include <vector>
#include <cstring>
#include <float.h>
struct whisper_mel {
int n_len;
int n_len_org;
int n_mel;
std::vector<float> data;
};
struct whisper_filters {
int32_t n_mel = 80;
int32_t n_fft = 201;
std::vector<float> data;
};
struct whisper_state {
int64_t t_sample_us = 0;
int64_t t_encode_us = 0;
int64_t t_decode_us = 0;
int64_t t_batchd_us = 0;
int64_t t_prompt_us = 0;
int64_t t_mel_us = 0;
int32_t n_sample = 0; // number of tokens sampled
int32_t n_encode = 0; // number of encoder calls
int32_t n_decode = 0; // number of decoder calls with n_tokens == 1 (text-generation)
int32_t n_batchd = 0; // number of decoder calls with n_tokens < 16 (batch decoding)
int32_t n_prompt = 0; // number of decoder calls with n_tokens > 1 (prompt encoding)
int32_t n_fail_p = 0; // number of logprob threshold failures
int32_t n_fail_h = 0; // number of entropy threshold failures
whisper_mel mel;
// decode output (2-dimensional array: [n_tokens][n_vocab])
std::vector<float> logits;
std::vector<whisper_token> prompt_past;
int lang_id = 0; // english by default
std::string path_model; // populated by whisper_init_from_file_with_params()
// [EXPERIMENTAL] token-level timestamps data
int64_t t_beg = 0;
int64_t t_last = 0;
whisper_token tid_last;
std::vector<float> energy; // PCM signal energy
// [EXPERIMENTAL] speed-up techniques
int32_t exp_n_audio_ctx = 0; // 0 - use default
};
struct whisper_model {
whisper_filters filters;
};
struct whisper_context {
int64_t t_load_us = 0;
int64_t t_start_us = 0;
whisper_model model;
whisper_vocab vocab;
whisper_state * state = nullptr;
};
template<typename T>
static void read_safe(whisper_model_loader * loader, T & dest) {
loader->read(loader->context, &dest, sizeof(T));
}
#endif // PRE_POST_COMMON_H

53
examples/whisper/cpp/src/pre_process_whisper.cpp Executable file
View file

@ -0,0 +1,53 @@
/*
* Copyright (C) 2024–2025 Amlogic, Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "pre_process_whisper.h"
#include "pre_post_common.h"
extern bool is_finish;
std::vector<float> do_pre_process(std::string fname_inp)
{
std::vector<float> pcmf32; // mono-channel F32 PCM
std::vector<std::vector<float>> pcmf32s; // stereo-channel F32 PCM
struct whisper_context ctx;
struct whisper_state state;
if (!read_wav(fname_inp, pcmf32, pcmf32s, false)) {
fprintf(stderr, "error: failed to read WAV file '%s'\n", fname_inp.c_str());
is_finish = true;
return {};
}
if (float(pcmf32.size())/WHISPER_SAMPLE_RATE == 0) {
is_finish = true;
return {};
}
if (whisper_pcm_to_mel_with_state(&ctx, &state, pcmf32.data(), pcmf32.size(), 8) != 0) {
printf("%s: failed to compute log mel spectrogram\n", __func__);
}
std::vector<float> input_data;
for (int j = 0; j < 80; j++) {
for (int i = 0; i < 3000; i++) {
input_data.push_back(state.mel.data[j * state.mel.n_len + i]);
}
}
return input_data;
}

21
examples/whisper/cpp/src/pre_process_whisper.h Executable file
View file

@ -0,0 +1,21 @@
/*
* Copyright (C) 2024–2025 Amlogic, Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <vector>
#include <string>
#include <float.h>
std::vector<float> do_pre_process(std::string fname_inp);

433
examples/whisper/cpp/src/whisper.cpp Executable file
View file

@ -0,0 +1,433 @@
/*
* Copyright (C) 2024–2025 Amlogic, Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "whisper.h"
#include <atomic>
#include <algorithm>
#define _USE_MATH_DEFINES
#include <cassert>
#include <cmath>
#include <cstdio>
#include <cstdarg>
#include <cstring>
#include <fstream>
#include <set>
#include <thread>
#include <vector>
#include <regex>
#include <random>
#include <functional>
#include <codecvt>
struct whisper_mel {
int n_len;
int n_len_org;
int n_mel;
std::vector<float> data;
};
struct whisper_filters {
int32_t n_mel = 80;
int32_t n_fft = 201;
std::vector<float> data;
};
struct whisper_state {
int64_t t_sample_us = 0;
int64_t t_encode_us = 0;
int64_t t_decode_us = 0;
int64_t t_batchd_us = 0;
int64_t t_prompt_us = 0;
int64_t t_mel_us = 0;
int32_t n_sample = 0; // number of tokens sampled
int32_t n_encode = 0; // number of encoder calls
int32_t n_decode = 0; // number of decoder calls with n_tokens == 1 (text-generation)
int32_t n_batchd = 0; // number of decoder calls with n_tokens < 16 (batch decoding)
int32_t n_prompt = 0; // number of decoder calls with n_tokens > 1 (prompt encoding)
int32_t n_fail_p = 0; // number of logprob threshold failures
int32_t n_fail_h = 0; // number of entropy threshold failures
whisper_mel mel;
std::vector<float> logits;
std::vector<whisper_token> prompt_past;
int lang_id = 0; // english by default
std::string path_model; // populated by whisper_init_from_file_with_params()
// [EXPERIMENTAL] token-level timestamps data
int64_t t_beg = 0;
int64_t t_last = 0;
whisper_token tid_last;
std::vector<float> energy; // PCM signal energy
// [EXPERIMENTAL] speed-up techniques
int32_t exp_n_audio_ctx = 0; // 0 - use default
};
struct whisper_model {
whisper_filters filters;
};
struct whisper_context {
int64_t t_load_us = 0;
int64_t t_start_us = 0;
whisper_model model;
whisper_vocab vocab;
whisper_state * state = nullptr;
};
#define SIN_COS_N_COUNT WHISPER_N_FFT
static float sin_vals[SIN_COS_N_COUNT];
static float cos_vals[SIN_COS_N_COUNT];
// In FFT, we frequently use sine and cosine operations with the same values.
// We can use precalculated values to speed up the process.
static void fill_sin_cos_table() {
static bool is_filled = false;
if (is_filled) return;
for (int i = 0; i < SIN_COS_N_COUNT; i++) {
double theta = (2*M_PI*i)/SIN_COS_N_COUNT;
sin_vals[i] = sinf(theta);
cos_vals[i] = cosf(theta);
}
is_filled = true;
}
// naive Discrete Fourier Transform
// input is real-valued
// output is complex-valued
static void dft(const std::vector<float> & in, std::vector<float> & out) {
int N = in.size();
out.resize(N*2);
const int sin_cos_step = SIN_COS_N_COUNT / N;
for (int k = 0; k < N; k++) {
float re = 0;
float im = 0;
for (int n = 0; n < N; n++) {
int idx = (k * n * sin_cos_step) % (SIN_COS_N_COUNT); // t = 2*M_PI*k*n/N
re += in[n]*cos_vals[idx]; // cos(t)
im -= in[n]*sin_vals[idx]; // sin(t)
}
out[k*2 + 0] = re;
out[k*2 + 1] = im;
}
}
// Cooley-Tukey FFT
// poor man's implementation - use something better
// input is real-valued
// output is complex-valued
static void fft(const std::vector<float> & in, std::vector<float> & out) {
out.resize(in.size()*2);
int N = in.size();
if (N == 1) {
out[0] = in[0];
out[1] = 0;
return;
}
if (N%2 == 1) {
dft(in, out);
return;
}
std::vector<float> even;
std::vector<float> odd;
even.reserve(N/2);
odd.reserve(N/2);
for (int i = 0; i < N; i++) {
if (i % 2 == 0) {
even.push_back(in[i]);
} else {
odd.push_back(in[i]);
}
}
std::vector<float> even_fft;
std::vector<float> odd_fft;
fft(even, even_fft);
fft(odd, odd_fft);
const int sin_cos_step = SIN_COS_N_COUNT / N;
for (int k = 0; k < N/2; k++) {
int idx = k * sin_cos_step; // t = 2*M_PI*k/N
float re = cos_vals[idx]; // cos(t)
float im = -sin_vals[idx]; // sin(t)
float re_odd = odd_fft[2*k + 0];
float im_odd = odd_fft[2*k + 1];
out[2*k + 0] = even_fft[2*k + 0] + re*re_odd - im*im_odd;
out[2*k + 1] = even_fft[2*k + 1] + re*im_odd + im*re_odd;
out[2*(k + N/2) + 0] = even_fft[2*k + 0] - re*re_odd + im*im_odd;
out[2*(k + N/2) + 1] = even_fft[2*k + 1] - re*im_odd - im*re_odd;
}
}
static bool hann_window(int length, bool periodic, std::vector<float> & output) {
if (output.size() < static_cast<size_t>(length)) {
output.resize(length);
}
int offset = -1;
if (periodic) {
offset = 0;
}
for (int i = 0; i < length; i++) {
output[i] = 0.5*(1.0 - cosf((2.0*M_PI*i)/(length + offset)));
}
return true;
}
static void log_mel_spectrogram_worker_thread(int ith, const std::vector<float> & hann, const std::vector<float> & samples,
int n_samples, int frame_size, int frame_step, int n_threads,
const whisper_filters & filters, whisper_mel & mel) {
std::vector<float> fft_in(frame_size, 0.0);
std::vector<float> fft_out(2 * frame_size);
int n_fft = filters.n_fft;
int i = ith;
assert(n_fft == 1 + (frame_size / 2));
// calculate FFT only when fft_in are not all zero
for (; i < std::min(n_samples / frame_step + 1, mel.n_len); i += n_threads) {
const int offset = i * frame_step;
// apply Hanning window (~10% faster)
for (int j = 0; j < std::min(frame_size, n_samples - offset); j++) {
fft_in[j] = hann[j] * samples[offset + j];
}
// fill the rest with zeros
if (n_samples - offset < frame_size) {
std::fill(fft_in.begin() + (n_samples - offset), fft_in.end(), 0.0);
}
// FFT
fft(fft_in, fft_out);
// Calculate modulus^2 of complex numbers
// Use pow(fft_out[2 * j + 0], 2) + pow(fft_out[2 * j + 1], 2) causes inference quality problem? Interesting.
for (int j = 0; j < n_fft; j++) {
fft_out[j] = (fft_out[2 * j + 0] * fft_out[2 * j + 0] + fft_out[2 * j + 1] * fft_out[2 * j + 1]);
}
// mel spectrogram
for (int j = 0; j < mel.n_mel; j++) {
double sum = 0.0;
// unroll loop (suggested by GH user @lunixbochs)
int k = 0;
for (k = 0; k < n_fft - 3; k += 4) {
sum +=
fft_out[k + 0] * filters.data[j * n_fft + k + 0] +
fft_out[k + 1] * filters.data[j * n_fft + k + 1] +
fft_out[k + 2] * filters.data[j * n_fft + k + 2] +
fft_out[k + 3] * filters.data[j * n_fft + k + 3];
}
// handle n_fft remainder
for (; k < n_fft; k++) {
sum += fft_out[k] * filters.data[j * n_fft + k];
}
sum = log10(std::max(sum, 1e-10));
mel.data[j * mel.n_len + i] = sum;
}
}
// Otherwise fft_out are all zero
double sum = log10(1e-10);
for (; i < mel.n_len; i += n_threads) {
for (int j = 0; j < mel.n_mel; j++) {
mel.data[j * mel.n_len + i] = sum;
}
}
}
static bool log_mel_spectrogram(
whisper_state & wstate,
const float * samples,
const int n_samples,
const int /*sample_rate*/,
const int frame_size,
const int frame_step,
const int n_mel,
const int n_threads,
whisper_filters & filters,
const bool debug,
whisper_mel & mel) {
// Hanning window (Use cosf to eliminate difference)
// ref: https://pytorch.org/docs/stable/generated/torch.hann_window.html
// ref: https://github.com/openai/whisper/blob/main/whisper/audio.py#L147
fill_sin_cos_table();
// auto & filters = filters;
filters.data.resize(filters.n_mel*filters.n_fft);
auto fin = std::ifstream("./data_bin/data.bin", std::ios::binary);
if (!fin)
{
fprintf(stderr, "%s : fail to open '%s'\n", __func__, "./data_bin/data.bin");
}
fin.read((char *)filters.data.data(), filters.data.size()*sizeof(float));
fin.eof();
fin.close();
std::vector<float> hann;
hann_window(frame_size, true, hann);
// Calculate the length of padding
int64_t stage_1_pad = WHISPER_SAMPLE_RATE * 30;
int64_t stage_2_pad = frame_size / 2;
// Initialize a vector and copy data from C array to it.
std::vector<float> samples_padded;
samples_padded.resize(n_samples + stage_1_pad + stage_2_pad * 2);
std::copy(samples, samples + n_samples, samples_padded.begin() + stage_2_pad);
// pad 30 seconds of zeros at the end of audio (480,000 samples) + reflective pad 200 samples at the end of audio
std::fill(samples_padded.begin() + n_samples + stage_2_pad, samples_padded.begin() + n_samples + stage_1_pad + 2 * stage_2_pad, 0);
// reflective pad 200 samples at the beginning of audio
std::reverse_copy(samples + 1, samples + 1 + stage_2_pad, samples_padded.begin());
mel.n_mel = n_mel;
// https://github.com/pytorch/pytorch/blob/main/aten/src/ATen/native/SpectralOps.cpp#L936
// Calculate number of frames + remove the last frame
mel.n_len = (samples_padded.size() - frame_size) / frame_step;
// Calculate semi-padded sample length to ensure compatibility
mel.n_len_org = 1 + (n_samples + stage_2_pad - frame_size) / frame_step;
mel.data.resize(mel.n_mel * mel.n_len);
{
std::vector<std::thread> workers(n_threads - 1);
for (int iw = 0; iw < n_threads - 1; ++iw) {
workers[iw] = std::thread(
log_mel_spectrogram_worker_thread, iw + 1, std::cref(hann), samples_padded,
n_samples + stage_2_pad, frame_size, frame_step, n_threads,
std::cref(filters), std::ref(mel));
}
// main thread
log_mel_spectrogram_worker_thread(0, hann, samples_padded, n_samples + stage_2_pad, frame_size, frame_step, n_threads, filters, mel);
for (int iw = 0; iw < n_threads - 1; ++iw) {
workers[iw].join();
}
}
// clamping and normalization
double mmax = -1e20;
for (int i = 0; i < mel.n_mel*mel.n_len; i++) {
if (mel.data[i] > mmax) {
mmax = mel.data[i];
}
}
mmax -= 8.0;
for (int i = 0; i < mel.n_mel*mel.n_len; i++) {
if (mel.data[i] < mmax) {
mel.data[i] = mmax;
}
mel.data[i] = (mel.data[i] + 4.0)/4.0;
}
return true;
}
int whisper_pcm_to_mel_with_state(struct whisper_context * ctx, struct whisper_state * state, const float * samples, int n_samples, int n_threads) {
if (!log_mel_spectrogram(*state, samples, n_samples, WHISPER_SAMPLE_RATE, WHISPER_N_FFT, WHISPER_HOP_LENGTH, 80, n_threads, ctx->model.filters, true, state->mel)) {
printf("%s: failed to compute mel spectrogram\n", __func__);
return -1;
}
return 0;
}
static std::vector<whisper_vocab::id> tokenize(const whisper_vocab & vocab, const std::string & text) {
std::vector<std::string> words;
// first split the text into words
{
std::string str = text;
std::string pat = R"('s|'t|'re|'ve|'m|'ll|'d| ?[[:alpha:]]+| ?[[:digit:]]+| ?[^\s[:alpha:][:digit:]]+|\s+(?!\S)|\s+)";
std::regex re(pat);
std::smatch m;
while (std::regex_search(str, m, re)) {
for (auto x : m) {
words.push_back(x);
}
str = m.suffix();
}
}
// find the longest tokens that form the words:
std::vector<whisper_vocab::id> tokens;
for (const auto & word : words) {
if (word.empty()) continue;
int i = 0;
int n = word.size();
while (i < n) {
int j = n;
bool found = false;
while (j > i) {
auto sub = word.substr(i, j-i);
auto it = vocab.token_to_id.find(sub);
if (it != vocab.token_to_id.end()) {
tokens.push_back(it->second);
i = j;
found = true;
break;
}
--j;
}
if (!found) {
printf("unknown token\n");
++i;
}
}
}
return tokens;
}

404
examples/whisper/cpp/src/whisper.h Executable file
View file

@ -0,0 +1,404 @@
/*
* Copyright (C) 2024–2025 Amlogic, Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef WHISPER_H
#define WHISPER_H
#include <stddef.h>
#include <stdint.h>
#include <stdbool.h>
#include <string>
#include <map>
#include <cstdint>
#ifdef __GNUC__
# define WHISPER_DEPRECATED(func, hint) func __attribute__((deprecated(hint)))
#elif defined(_MSC_VER)
# define WHISPER_DEPRECATED(func, hint) __declspec(deprecated(hint)) func
#else
# define WHISPER_DEPRECATED(func, hint) func
#endif
#ifdef WHISPER_SHARED
# ifdef _WIN32
# ifdef WHISPER_BUILD
# define WHISPER_API __declspec(dllexport)
# else
# define WHISPER_API __declspec(dllimport)
# endif
# else
# define WHISPER_API __attribute__ ((visibility ("default")))
# endif
#else
# define WHISPER_API
#endif
#define WHISPER_SAMPLE_RATE 16000
#define WHISPER_N_FFT 400
#define WHISPER_HOP_LENGTH 160
#define WHISPER_CHUNK_SIZE 30
#define WHISPER_N_MELS 80
#define WHISPER_N_FRAMES 3000
#define WHISPER_N_SAMPLES 48000
struct whisper_vocab {
using id = int32_t;
using token = std::string;
int n_vocab = 51864;
std::map<token, id> token_to_id;
std::map<id, token> id_to_token;
// reference: https://github.com/openai/whisper/blob/248b6cb124225dd263bb9bd32d060b6517e067f8/whisper/tokenizer.py#L334-L349
id token_eot = 50256;
id token_sot = 50257;
// task tokens (used only for multilingual models)
id token_translate = 50357;
id token_transcribe = 50358;
// other special tokens
id token_solm = 50359; // [TDRZ] used by tinydiarize models to indicate speaker turn
id token_prev = 50360;
id token_nosp = 50361;
id token_not = 50362; // no timestamps
id token_beg = 50363; // begin timestamps
bool is_multilingual() const {
return n_vocab >= 51865;
}
int num_languages() const {
return n_vocab - 51765 - (is_multilingual() ? 1 : 0);
}
};
#ifdef __cplusplus
extern "C" {
#endif
struct whisper_context;
struct whisper_state;
struct whisper_full_params;
typedef int32_t whisper_pos;
typedef int32_t whisper_token;
typedef int32_t whisper_seq_id;
typedef struct whisper_token_data {
whisper_token id; // token id
whisper_token tid; // forced timestamp token id
float p; // probability of the token
float plog; // log probability of the token
float pt; // probability of the timestamp token
float ptsum; // sum of probabilities of all timestamp tokens
// token-level timestamp data
// do not use if you haven't computed token-level timestamps
int64_t t0; // start time of the token
int64_t t1; // end time of the token
// [EXPERIMENTAL] Token-level timestamps with DTW
// do not use if you haven't computed token-level timestamps with dtw
// Roughly corresponds to the moment in audio in which the token was output
int64_t t_dtw;
float vlen; // voice length of the token
} whisper_token_data;
typedef struct whisper_model_loader {
void * context;
size_t (*read)(void * ctx, void * output, size_t read_size);
bool (*eof)(void * ctx);
void (*close)(void * ctx);
} whisper_model_loader;
// Various functions for loading a ggml whisper model.
// Allocate (almost) all memory needed for the model.
// Return NULL on failure
WHISPER_API struct whisper_context * whisper_init_from_file_with_params (const char * path_model, struct whisper_context_params params);
WHISPER_API struct whisper_context * whisper_init_from_buffer_with_params(void * buffer, size_t buffer_size, struct whisper_context_params params);
WHISPER_API struct whisper_context * whisper_init_with_params (struct whisper_model_loader * loader, struct whisper_context_params params);
// These are the same as the above, but the internal state of the context is not allocated automatically
// It is the responsibility of the caller to allocate the state using whisper_init_state() (#523)
WHISPER_API struct whisper_context * whisper_init_from_file_with_params_no_state (const char * path_model, struct whisper_context_params params);
WHISPER_API struct whisper_context * whisper_init_from_buffer_with_params_no_state(void * buffer, size_t buffer_size, struct whisper_context_params params);
WHISPER_API struct whisper_context * whisper_init_with_params_no_state (struct whisper_model_loader * loader, struct whisper_context_params params);
WHISPER_API struct whisper_state * whisper_init_state(struct whisper_context * ctx);
// Given a context, enable use of OpenVINO for encode inference.
// model_path: Optional path to OpenVINO encoder IR model. If set to nullptr,
// the path will be generated from the ggml model path that was passed
// in to whisper_init_from_file. For example, if 'path_model' was
// "/path/to/ggml-base.en.bin", then OpenVINO IR model path will be
// assumed to be "/path/to/ggml-base.en-encoder-openvino.xml".
// device: OpenVINO device to run inference on ("CPU", "GPU", etc.)
// cache_dir: Optional cache directory that can speed up init time, especially for
// GPU, by caching compiled 'blobs' there.
// Set to nullptr if not used.
// Returns 0 on success. If OpenVINO is not enabled in build, this simply returns 1.
WHISPER_API int whisper_ctx_init_openvino_encoder(
struct whisper_context * ctx,
const char * model_path,
const char * device,
const char * cache_dir);
// Frees all allocated memory
WHISPER_API void whisper_free (struct whisper_context * ctx);
WHISPER_API void whisper_free_state(struct whisper_state * state);
WHISPER_API void whisper_free_params(struct whisper_full_params * params);
WHISPER_API void whisper_free_context_params(struct whisper_context_params * params);
// Convert RAW PCM audio to log mel spectrogram.
// The resulting spectrogram is stored inside the default state of the provided whisper context.
// Returns 0 on success
WHISPER_API int whisper_pcm_to_mel(
struct whisper_context * ctx,
const float * samples,
int n_samples,
int n_threads);
WHISPER_API int whisper_pcm_to_mel_with_state(
struct whisper_context * ctx,
struct whisper_state * state,
const float * samples,
int n_samples,
int n_threads);
// Convert RAW PCM audio to log mel spectrogram but applies a Phase Vocoder to speed up the audio x2.
// The resulting spectrogram is stored inside the default state of the provided whisper context.
// Returns 0 on success
WHISPER_API int whisper_pcm_to_mel_phase_vocoder(
struct whisper_context * ctx,
const float * samples,
int n_samples,
int n_threads);
WHISPER_API int whisper_pcm_to_mel_phase_vocoder_with_state(
struct whisper_context * ctx,
struct whisper_state * state,
const float * samples,
int n_samples,
int n_threads);
// This can be used to set a custom log mel spectrogram inside the default state of the provided whisper context.
// Use this instead of whisper_pcm_to_mel() if you want to provide your own log mel spectrogram.
// n_mel must be 80
// Returns 0 on success
WHISPER_API int whisper_set_mel(
struct whisper_context * ctx,
const float * data,
int n_len,
int n_mel);
WHISPER_API int whisper_set_mel_with_state(
struct whisper_context * ctx,
struct whisper_state * state,
const float * data,
int n_len,
int n_mel);
// Run the Whisper encoder on the log mel spectrogram stored inside the default state in the provided whisper context.
// Make sure to call whisper_pcm_to_mel() or whisper_set_mel() first.
// offset can be used to specify the offset of the first frame in the spectrogram.
// Returns 0 on success
WHISPER_API int whisper_encode(
struct whisper_context * ctx,
int offset,
int n_threads);
WHISPER_API int whisper_encode_with_state(
struct whisper_context * ctx,
struct whisper_state * state,
int offset,
int n_threads);
// Run the Whisper decoder to obtain the logits and probabilities for the next token.
// Make sure to call whisper_encode() first.
// tokens + n_tokens is the provided context for the decoder.
// n_past is the number of tokens to use from previous decoder calls.
// Returns 0 on success
// TODO: add support for multiple decoders
WHISPER_API int whisper_decode(
struct whisper_context * ctx,
const whisper_token * tokens,
int n_tokens,
int n_past,
int n_threads);
WHISPER_API int whisper_decode_with_state(
struct whisper_context * ctx,
struct whisper_state * state,
const whisper_token * tokens,
int n_tokens,
int n_past,
int n_threads);
// Convert the provided text into tokens.
// The tokens pointer must be large enough to hold the resulting tokens.
// Returns the number of tokens on success, no more than n_max_tokens
// Returns a negative number on failure - the number of tokens that would have been returned
// TODO: not sure if correct
WHISPER_API int whisper_tokenize(
struct whisper_context * ctx,
const char * text,
whisper_token * tokens,
int n_max_tokens);
// Return the number of tokens in the provided text
// Equivalent to: -whisper_tokenize(ctx, text, NULL, 0)
int whisper_token_count(struct whisper_context * ctx, const char * text);
// Largest language id (i.e. number of available languages - 1)
WHISPER_API int whisper_lang_max_id();
// Return the id of the specified language, returns -1 if not found
// Examples:
// "de" -> 2
// "german" -> 2
WHISPER_API int whisper_lang_id(const char * lang);
// Return the short string of the specified language id (e.g. 2 -> "de"), returns nullptr if not found
WHISPER_API const char * whisper_lang_str(int id);
// Return the short string of the specified language name (e.g. 2 -> "german"), returns nullptr if not found
WHISPER_API const char * whisper_lang_str_full(int id);
// Use mel data at offset_ms to try and auto-detect the spoken language
// Make sure to call whisper_pcm_to_mel() or whisper_set_mel() first
// Returns the top language id or negative on failure
// If not null, fills the lang_probs array with the probabilities of all languages
// The array must be whisper_lang_max_id() + 1 in size
// ref: https://github.com/openai/whisper/blob/main/whisper/decoding.py#L18-L69
WHISPER_API int whisper_lang_auto_detect(
struct whisper_context * ctx,
int offset_ms,
int n_threads,
float * lang_probs);
WHISPER_API int whisper_lang_auto_detect_with_state(
struct whisper_context * ctx,
struct whisper_state * state,
int offset_ms,
int n_threads,
float * lang_probs);
WHISPER_API int whisper_n_len (struct whisper_context * ctx); // mel length
WHISPER_API int whisper_n_len_from_state(struct whisper_state * state); // mel length
WHISPER_API int whisper_n_vocab (struct whisper_context * ctx);
WHISPER_API int whisper_n_text_ctx (struct whisper_context * ctx);
WHISPER_API int whisper_n_audio_ctx (struct whisper_context * ctx);
WHISPER_API int whisper_is_multilingual (struct whisper_context * ctx);
WHISPER_API int whisper_model_n_vocab (struct whisper_context * ctx);
WHISPER_API int whisper_model_n_audio_ctx (struct whisper_context * ctx);
WHISPER_API int whisper_model_n_audio_state(struct whisper_context * ctx);
WHISPER_API int whisper_model_n_audio_head (struct whisper_context * ctx);
WHISPER_API int whisper_model_n_audio_layer(struct whisper_context * ctx);
WHISPER_API int whisper_model_n_text_ctx (struct whisper_context * ctx);
WHISPER_API int whisper_model_n_text_state (struct whisper_context * ctx);
WHISPER_API int whisper_model_n_text_head (struct whisper_context * ctx);
WHISPER_API int whisper_model_n_text_layer (struct whisper_context * ctx);
WHISPER_API int whisper_model_n_mels (struct whisper_context * ctx);
WHISPER_API int whisper_model_ftype (struct whisper_context * ctx);
WHISPER_API int whisper_model_type (struct whisper_context * ctx);
// Token logits obtained from the last call to whisper_decode()
// The logits for the last token are stored in the last row
// Rows: n_tokens
// Cols: n_vocab
WHISPER_API float * whisper_get_logits (struct whisper_context * ctx);
WHISPER_API float * whisper_get_logits_from_state(struct whisper_state * state);
// Token Id -> String. Uses the vocabulary in the provided context
WHISPER_API const char * whisper_token_to_str(struct whisper_context * ctx, whisper_token token);
WHISPER_API const char * whisper_model_type_readable(struct whisper_context * ctx);
// Special tokens
WHISPER_API whisper_token whisper_token_eot (struct whisper_context * ctx);
WHISPER_API whisper_token whisper_token_sot (struct whisper_context * ctx);
WHISPER_API whisper_token whisper_token_solm(struct whisper_context * ctx);
WHISPER_API whisper_token whisper_token_prev(struct whisper_context * ctx);
WHISPER_API whisper_token whisper_token_nosp(struct whisper_context * ctx);
WHISPER_API whisper_token whisper_token_not (struct whisper_context * ctx);
WHISPER_API whisper_token whisper_token_beg (struct whisper_context * ctx);
WHISPER_API whisper_token whisper_token_lang(struct whisper_context * ctx, int lang_id);
// Task tokens
WHISPER_API whisper_token whisper_token_translate (struct whisper_context * ctx);
WHISPER_API whisper_token whisper_token_transcribe(struct whisper_context * ctx);
////////////////////////////////////////////////////////////////////////////
// Number of generated text segments
// A segment can be a few words, a sentence, or even a paragraph.
WHISPER_API int whisper_full_n_segments (struct whisper_context * ctx);
WHISPER_API int whisper_full_n_segments_from_state(struct whisper_state * state);
// Language id associated with the context's default state
WHISPER_API int whisper_full_lang_id(struct whisper_context * ctx);
// Language id associated with the provided state
WHISPER_API int whisper_full_lang_id_from_state(struct whisper_state * state);
// Get the start and end time of the specified segment
WHISPER_API int64_t whisper_full_get_segment_t0 (struct whisper_context * ctx, int i_segment);
WHISPER_API int64_t whisper_full_get_segment_t0_from_state(struct whisper_state * state, int i_segment);
WHISPER_API int64_t whisper_full_get_segment_t1 (struct whisper_context * ctx, int i_segment);
WHISPER_API int64_t whisper_full_get_segment_t1_from_state(struct whisper_state * state, int i_segment);
// Get whether the next segment is predicted as a speaker turn
WHISPER_API bool whisper_full_get_segment_speaker_turn_next(struct whisper_context * ctx, int i_segment);
WHISPER_API bool whisper_full_get_segment_speaker_turn_next_from_state(struct whisper_state * state, int i_segment);
// Get the text of the specified segment
WHISPER_API const char * whisper_full_get_segment_text (struct whisper_context * ctx, int i_segment);
WHISPER_API const char * whisper_full_get_segment_text_from_state(struct whisper_state * state, int i_segment);
// Get number of tokens in the specified segment
WHISPER_API int whisper_full_n_tokens (struct whisper_context * ctx, int i_segment);
WHISPER_API int whisper_full_n_tokens_from_state(struct whisper_state * state, int i_segment);
// Get the token text of the specified token in the specified segment
WHISPER_API const char * whisper_full_get_token_text (struct whisper_context * ctx, int i_segment, int i_token);
WHISPER_API const char * whisper_full_get_token_text_from_state(struct whisper_context * ctx, struct whisper_state * state, int i_segment, int i_token);
WHISPER_API whisper_token whisper_full_get_token_id (struct whisper_context * ctx, int i_segment, int i_token);
WHISPER_API whisper_token whisper_full_get_token_id_from_state(struct whisper_state * state, int i_segment, int i_token);
// Get token data for the specified token in the specified segment
// This contains probabilities, timestamps, etc.
WHISPER_API whisper_token_data whisper_full_get_token_data (struct whisper_context * ctx, int i_segment, int i_token);
WHISPER_API whisper_token_data whisper_full_get_token_data_from_state(struct whisper_state * state, int i_segment, int i_token);
// Get the probability of the specified token in the specified segment
WHISPER_API float whisper_full_get_token_p (struct whisper_context * ctx, int i_segment, int i_token);
WHISPER_API float whisper_full_get_token_p_from_state(struct whisper_state * state, int i_segment, int i_token);
////////////////////////////////////////////////////////////////////////////
#ifdef __cplusplus
}
#endif
#endif

306
examples/whisper/cpp/src/whisper_invoke.cpp Executable file
View file

@ -0,0 +1,306 @@
/*
* Copyright (C) 2024–2025 Amlogic, Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*-------------------------------------------
Includes
-------------------------------------------*/
#include <stdio.h>
#include <string.h>
#include <iostream>
#include <algorithm>
#include "nn_sdk.h"
#include "whisper.h"
#include "whisper_invoke.h"
struct DMAConfig {
bool use_dma = true;
bool malloc_buffer_once = true;
};
DMAConfig encoder, decoder;
bool is_finish = false;
static int decoder_input_1_size = 2; /* init decoder input_1 size*/
whisper_vocab vocab_out_init;
///////////////////////////////////////////////////////////
#define TIKTOKEN_ID_STOP 50256
#define INPUT_SHAPE 48
#define decoder_model_inputs_size 2
aml_memory_config_t mem_config_encoder;
aml_memory_data_t mem_data_encoder;
aml_memory_config_t mem_config[decoder_model_inputs_size];
aml_memory_data_t mem_data[decoder_model_inputs_size];
whisper_vocab read_token_info(std::string token_path);
void* init_network_file(const char *model_path)
{
void *qcontext = NULL;
aml_config config;
static bool is_worker_initialized = false;
if (!is_worker_initialized) {
vocab_out_init = read_token_info("./data_bin/tokenizer_info.bin");
is_worker_initialized = true;
}
memset(&config, 0, sizeof(aml_config));
config.nbgType = NN_ADLA_FILE;
config.path = model_path;
config.modelType = ADLA_LOADABLE;
config.typeSize = sizeof(aml_config);
/* set omp, If you are considering high CPU usage during operation,
you can turn off this api, set_openmp_opt_flag = false */
aml_openmp_opt_t openmp_opt[] =
{
{
.operator_type = AML_Unknown,
.enable_openmp = true,
.involve_all_ops = true,
.openmp_num = 2,
},
};
config.forward_ctrl.softop_info.set_openmp_opt_flag = true;
config.forward_ctrl.softop_info.openmp_opt_num = sizeof(openmp_opt) / sizeof(aml_openmp_opt_t);
config.forward_ctrl.softop_info.openmp_opt = openmp_opt;
/* set neon */
aml_neon_opt_t neon_opt[] =
{
{
.operator_type = AML_Unknown,
.enable_neon = true,
.involve_all_ops = true,
},
};
config.forward_ctrl.softop_info.set_neon_opt_flag = true;
config.forward_ctrl.softop_info.neon_opt_num = sizeof(neon_opt) / sizeof(aml_neon_opt_t);
config.forward_ctrl.softop_info.neon_opt = neon_opt;
qcontext = aml_module_create(&config);
if (NULL == qcontext)
{
printf("aml_module_create fail.\n");
return NULL;
}
return qcontext;
}
bool is_finish_end() {
return is_finish;
}
std::vector<float> run_network_encoder_process(void *qcontext, std::vector<float> input_ids)
{
int ret = 0;
nn_input inData;
size_t outputData_size;
nn_output *outdata = NULL;
aml_output_config_t outconfig;
is_finish = false; /* init is_finish -> false */
inData.input_index = 0;
inData.info.input_format = AML_INPUT_DEFAULT;
inData.size = input_ids.size() * sizeof(float); /* INPUT_SHAPE --->>> input_ids.size() */
if (encoder.use_dma) {
if (encoder.malloc_buffer_once) {
mem_config_encoder.cache_type = AML_WITH_CACHE;
mem_config_encoder.memory_type = AML_VIRTUAL_ADDR;
mem_config_encoder.direction = AML_MEM_DIRECTION_READ_WRITE;
mem_config_encoder.index = 0;
mem_config_encoder.mem_size = inData.size;
aml_util_mallocBuffer(qcontext, &mem_config_encoder, &mem_data_encoder);
aml_util_swapExternalInputBuffer(qcontext, &mem_config_encoder, &mem_data_encoder);
}
inData.input_type = INPUT_DMA_DATA;
memcpy(mem_data_encoder.viraddr, input_ids.data(), mem_config_encoder.mem_size);
inData.input = NULL;
} else {
inData.input = reinterpret_cast<unsigned char*>(input_ids.data());
inData.input_type = BINARY_RAW_DATA;
ret = aml_module_input_set(qcontext, &inData);
if (ret)
{
printf("aml_module_input_set fail.\n");
}
}
encoder.malloc_buffer_once = false;
memset(&outconfig, 0, sizeof(aml_output_config_t));
if (encoder.use_dma) {
outconfig.format = AML_OUTDATA_DMA;
} else {
outconfig.format = AML_OUTDATA_RAW;
}
outconfig.typeSize = sizeof(aml_output_config_t);
outdata = (nn_output*)aml_module_output_get(qcontext, outconfig);
outputData_size = outdata->out[0].size / sizeof(float);
std::vector<float> buf_data(reinterpret_cast<float*>(outdata->out[0].buf), reinterpret_cast<float*>(outdata->out[0].buf) + outputData_size);
return buf_data;
}
nn_output* run_network_decoder_process(void *qcontext, Input_Decoder* input_data)
{
int ret = 0;
nn_input inData;
nn_output *outdata = NULL;
aml_output_config_t outconfig;
for (int i = 0; i < decoder_model_inputs_size; i++)
{
inData.input_index = i;
inData.info.input_format = AML_INPUT_DEFAULT;
inData.size = i == 0 ? input_data->input_0_size * sizeof(float) : input_data->input_1_size * sizeof(int64_t);
if (decoder.use_dma) {
if (decoder.malloc_buffer_once) {
mem_config[i].index = i;
mem_config[i].mem_size = inData.size;
mem_config[i].cache_type = AML_WITH_CACHE;
mem_config[i].memory_type = AML_VIRTUAL_ADDR;
mem_config[i].direction = AML_MEM_DIRECTION_READ_WRITE;
aml_util_mallocBuffer(qcontext, &mem_config[i], &mem_data[i]);
aml_util_swapExternalInputBuffer(qcontext, &mem_config[i], &mem_data[i]);
}
inData.input_type = INPUT_DMA_DATA;
memcpy(mem_data[i].viraddr, i == 0 ? static_cast<const void*>(input_data->input_0) :
static_cast<const void*>(input_data->input_1), mem_config[i].mem_size);
inData.input = NULL;
} else {
inData.input = i == 0 ? reinterpret_cast<unsigned char*>(const_cast<float*>(input_data->input_0)) :
reinterpret_cast<unsigned char*>(const_cast<int64_t*>(input_data->input_1));
inData.input_type = BINARY_RAW_DATA;
ret = aml_module_input_set(qcontext, &inData);
if (ret)
{
printf("aml_module_input_set fail.\n");
}
}
}
decoder.malloc_buffer_once = false;
memset(&outconfig, 0, sizeof(aml_output_config_t));
if (decoder.use_dma) {
outconfig.format = AML_OUTDATA_DMA;
} else {
outconfig.format = AML_OUTDATA_RAW;
}
outconfig.typeSize = sizeof(aml_output_config_t);
outdata = (nn_output*)aml_module_output_get(qcontext, outconfig);
return outdata;
}
std::string run_network_decoder(void *qcontext_sec, Input_Decoder* input_data)
{
int ret = 0;
int max_index = 0;
std::string out;
size_t id_shape, begin_count, last_count;
nn_output* buf_data_sec;
buf_data_sec = run_network_decoder_process(qcontext_sec, input_data);
float* buf_data = reinterpret_cast<float*>(buf_data_sec->out[0].buf);
id_shape = decoder_input_1_size;
begin_count = (id_shape - 1) * 51864; // why id_shape -1? output[0] shape [1, 64, 51864], save [id_shape - 1] group data
last_count = id_shape * 51864 - 1;
// get max_valus and max_index
auto max_it = std::max_element(buf_data + begin_count, buf_data + last_count);
max_index = std::distance(buf_data + begin_count, max_it);
input_data->input_1[id_shape] = max_index;
if (max_index == TIKTOKEN_ID_STOP || id_shape >= INPUT_SHAPE) {
is_finish = true;
if (max_index != TIKTOKEN_ID_STOP)
out = vocab_out_init.id_to_token.at(max_index).c_str();
decoder_input_1_size = 2;
}
else {
out = vocab_out_init.id_to_token.at(max_index).c_str();
decoder_input_1_size++;
}
return out;
}
int destroy_network(void *qcontext)
{
int ret = 0;
/* free encoder
encoder.use_dma = true
encoder.malloc_buffer_once = false
*/
if (encoder.use_dma && mem_config_encoder.mem_size != 0) {
ret = aml_util_freeBuffer(qcontext, &mem_config_encoder, &mem_data_encoder);
if (ret)
{
std::cout << "aml_util_freeBuffer fail." << std::endl;
}
}
encoder.use_dma = false;
/* free decoder
first use destroy_network, decoder.malloc_buffer_once is false,
and set decoder.malloc_buffer_once is true
*/
if (decoder.malloc_buffer_once && mem_config[0].mem_size != 0) {
for (int i = 0; i < decoder_model_inputs_size; i++)
{
ret = aml_util_freeBuffer(qcontext, &mem_config[i], &mem_data[i]);
if (ret)
{
std::cout << "aml_util_freeBuffer fail." << std::endl;
}
}
}
decoder.malloc_buffer_once = true;
ret = aml_module_destroy(qcontext);
if (ret)
{
printf("aml_module_destroy fail.\n");
return -1;
}
return ret;
}

39
examples/whisper/cpp/src/whisper_invoke.h Executable file
View file

@ -0,0 +1,39 @@
/*
* Copyright (C) 2024–2025 Amlogic, Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef WHISPER_INVOKE_H
#define WHISPER_INVOKE_H
#include <string>
#include <vector>
#include <map>
#include "nn_sdk.h"
struct Input_Decoder {
float * input_0;
int input_0_size;
int64_t * input_1;
int input_1_size;
};
void* init_network_file(const char *model_path);
std::vector<float> do_pre_process(std::string fname_inp);
std::vector<float> run_network_encoder_process(void *qcontext, std::vector<float> input_ids);
std::string run_network_decoder(void *qcontext_sec, Input_Decoder* input_data);
bool is_finish_end();
int destroy_network(void *qcontext);
#endif // WHISPER_INVOKE_H