433 lines
14 KiB
C++
Executable file
433 lines
14 KiB
C++
Executable file
/*
|
||
* Copyright (C) 2024–2025 Amlogic, Inc. All rights reserved.
|
||
*
|
||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||
* you may not use this file except in compliance with the License.
|
||
* You may obtain a copy of the License at
|
||
*
|
||
* http://www.apache.org/licenses/LICENSE-2.0
|
||
*
|
||
* Unless required by applicable law or agreed to in writing, software
|
||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
* See the License for the specific language governing permissions and
|
||
* limitations under the License.
|
||
*/
|
||
|
||
#include "whisper.h"
|
||
|
||
#include <atomic>
|
||
#include <algorithm>
|
||
#define _USE_MATH_DEFINES
|
||
#include <cassert>
|
||
#include <cmath>
|
||
#include <cstdio>
|
||
#include <cstdarg>
|
||
#include <cstring>
|
||
#include <fstream>
|
||
#include <set>
|
||
#include <thread>
|
||
#include <vector>
|
||
#include <regex>
|
||
#include <random>
|
||
#include <functional>
|
||
#include <codecvt>
|
||
|
||
struct whisper_mel {
|
||
int n_len;
|
||
int n_len_org;
|
||
int n_mel;
|
||
|
||
std::vector<float> data;
|
||
};
|
||
|
||
struct whisper_filters {
|
||
int32_t n_mel = 80;
|
||
int32_t n_fft = 201;
|
||
|
||
std::vector<float> data;
|
||
};
|
||
|
||
struct whisper_state {
|
||
int64_t t_sample_us = 0;
|
||
int64_t t_encode_us = 0;
|
||
int64_t t_decode_us = 0;
|
||
int64_t t_batchd_us = 0;
|
||
int64_t t_prompt_us = 0;
|
||
int64_t t_mel_us = 0;
|
||
|
||
int32_t n_sample = 0; // number of tokens sampled
|
||
int32_t n_encode = 0; // number of encoder calls
|
||
int32_t n_decode = 0; // number of decoder calls with n_tokens == 1 (text-generation)
|
||
int32_t n_batchd = 0; // number of decoder calls with n_tokens < 16 (batch decoding)
|
||
int32_t n_prompt = 0; // number of decoder calls with n_tokens > 1 (prompt encoding)
|
||
int32_t n_fail_p = 0; // number of logprob threshold failures
|
||
int32_t n_fail_h = 0; // number of entropy threshold failures
|
||
|
||
whisper_mel mel;
|
||
|
||
std::vector<float> logits;
|
||
std::vector<whisper_token> prompt_past;
|
||
|
||
int lang_id = 0; // english by default
|
||
|
||
std::string path_model; // populated by whisper_init_from_file_with_params()
|
||
|
||
// [EXPERIMENTAL] token-level timestamps data
|
||
int64_t t_beg = 0;
|
||
int64_t t_last = 0;
|
||
|
||
whisper_token tid_last;
|
||
|
||
std::vector<float> energy; // PCM signal energy
|
||
|
||
// [EXPERIMENTAL] speed-up techniques
|
||
int32_t exp_n_audio_ctx = 0; // 0 - use default
|
||
};
|
||
|
||
struct whisper_model {
|
||
whisper_filters filters;
|
||
};
|
||
|
||
struct whisper_context {
|
||
int64_t t_load_us = 0;
|
||
int64_t t_start_us = 0;
|
||
|
||
whisper_model model;
|
||
whisper_vocab vocab;
|
||
|
||
whisper_state * state = nullptr;
|
||
};
|
||
|
||
|
||
#define SIN_COS_N_COUNT WHISPER_N_FFT
|
||
static float sin_vals[SIN_COS_N_COUNT];
|
||
static float cos_vals[SIN_COS_N_COUNT];
|
||
|
||
// In FFT, we frequently use sine and cosine operations with the same values.
|
||
// We can use precalculated values to speed up the process.
|
||
static void fill_sin_cos_table() {
|
||
static bool is_filled = false;
|
||
if (is_filled) return;
|
||
for (int i = 0; i < SIN_COS_N_COUNT; i++) {
|
||
double theta = (2*M_PI*i)/SIN_COS_N_COUNT;
|
||
sin_vals[i] = sinf(theta);
|
||
cos_vals[i] = cosf(theta);
|
||
}
|
||
is_filled = true;
|
||
}
|
||
|
||
// naive Discrete Fourier Transform
|
||
// input is real-valued
|
||
// output is complex-valued
|
||
static void dft(const std::vector<float> & in, std::vector<float> & out) {
|
||
int N = in.size();
|
||
|
||
out.resize(N*2);
|
||
const int sin_cos_step = SIN_COS_N_COUNT / N;
|
||
|
||
for (int k = 0; k < N; k++) {
|
||
float re = 0;
|
||
float im = 0;
|
||
|
||
for (int n = 0; n < N; n++) {
|
||
int idx = (k * n * sin_cos_step) % (SIN_COS_N_COUNT); // t = 2*M_PI*k*n/N
|
||
re += in[n]*cos_vals[idx]; // cos(t)
|
||
im -= in[n]*sin_vals[idx]; // sin(t)
|
||
}
|
||
|
||
out[k*2 + 0] = re;
|
||
out[k*2 + 1] = im;
|
||
}
|
||
}
|
||
|
||
// Cooley-Tukey FFT
|
||
// poor man's implementation - use something better
|
||
// input is real-valued
|
||
// output is complex-valued
|
||
static void fft(const std::vector<float> & in, std::vector<float> & out) {
|
||
out.resize(in.size()*2);
|
||
|
||
int N = in.size();
|
||
|
||
if (N == 1) {
|
||
out[0] = in[0];
|
||
out[1] = 0;
|
||
return;
|
||
}
|
||
|
||
if (N%2 == 1) {
|
||
dft(in, out);
|
||
return;
|
||
}
|
||
|
||
std::vector<float> even;
|
||
std::vector<float> odd;
|
||
|
||
even.reserve(N/2);
|
||
odd.reserve(N/2);
|
||
|
||
for (int i = 0; i < N; i++) {
|
||
if (i % 2 == 0) {
|
||
even.push_back(in[i]);
|
||
} else {
|
||
odd.push_back(in[i]);
|
||
}
|
||
}
|
||
|
||
std::vector<float> even_fft;
|
||
std::vector<float> odd_fft;
|
||
|
||
fft(even, even_fft);
|
||
fft(odd, odd_fft);
|
||
|
||
const int sin_cos_step = SIN_COS_N_COUNT / N;
|
||
for (int k = 0; k < N/2; k++) {
|
||
int idx = k * sin_cos_step; // t = 2*M_PI*k/N
|
||
float re = cos_vals[idx]; // cos(t)
|
||
float im = -sin_vals[idx]; // sin(t)
|
||
|
||
float re_odd = odd_fft[2*k + 0];
|
||
float im_odd = odd_fft[2*k + 1];
|
||
|
||
out[2*k + 0] = even_fft[2*k + 0] + re*re_odd - im*im_odd;
|
||
out[2*k + 1] = even_fft[2*k + 1] + re*im_odd + im*re_odd;
|
||
|
||
out[2*(k + N/2) + 0] = even_fft[2*k + 0] - re*re_odd + im*im_odd;
|
||
out[2*(k + N/2) + 1] = even_fft[2*k + 1] - re*im_odd - im*re_odd;
|
||
}
|
||
}
|
||
|
||
static bool hann_window(int length, bool periodic, std::vector<float> & output) {
|
||
if (output.size() < static_cast<size_t>(length)) {
|
||
output.resize(length);
|
||
}
|
||
int offset = -1;
|
||
if (periodic) {
|
||
offset = 0;
|
||
}
|
||
for (int i = 0; i < length; i++) {
|
||
output[i] = 0.5*(1.0 - cosf((2.0*M_PI*i)/(length + offset)));
|
||
}
|
||
|
||
return true;
|
||
}
|
||
|
||
static void log_mel_spectrogram_worker_thread(int ith, const std::vector<float> & hann, const std::vector<float> & samples,
|
||
int n_samples, int frame_size, int frame_step, int n_threads,
|
||
const whisper_filters & filters, whisper_mel & mel) {
|
||
std::vector<float> fft_in(frame_size, 0.0);
|
||
std::vector<float> fft_out(2 * frame_size);
|
||
int n_fft = filters.n_fft;
|
||
int i = ith;
|
||
|
||
assert(n_fft == 1 + (frame_size / 2));
|
||
|
||
// calculate FFT only when fft_in are not all zero
|
||
for (; i < std::min(n_samples / frame_step + 1, mel.n_len); i += n_threads) {
|
||
const int offset = i * frame_step;
|
||
|
||
// apply Hanning window (~10% faster)
|
||
for (int j = 0; j < std::min(frame_size, n_samples - offset); j++) {
|
||
fft_in[j] = hann[j] * samples[offset + j];
|
||
}
|
||
// fill the rest with zeros
|
||
if (n_samples - offset < frame_size) {
|
||
std::fill(fft_in.begin() + (n_samples - offset), fft_in.end(), 0.0);
|
||
}
|
||
|
||
// FFT
|
||
fft(fft_in, fft_out);
|
||
|
||
// Calculate modulus^2 of complex numbers
|
||
// Use pow(fft_out[2 * j + 0], 2) + pow(fft_out[2 * j + 1], 2) causes inference quality problem? Interesting.
|
||
for (int j = 0; j < n_fft; j++) {
|
||
fft_out[j] = (fft_out[2 * j + 0] * fft_out[2 * j + 0] + fft_out[2 * j + 1] * fft_out[2 * j + 1]);
|
||
}
|
||
|
||
// mel spectrogram
|
||
for (int j = 0; j < mel.n_mel; j++) {
|
||
double sum = 0.0;
|
||
|
||
// unroll loop (suggested by GH user @lunixbochs)
|
||
int k = 0;
|
||
for (k = 0; k < n_fft - 3; k += 4) {
|
||
sum +=
|
||
fft_out[k + 0] * filters.data[j * n_fft + k + 0] +
|
||
fft_out[k + 1] * filters.data[j * n_fft + k + 1] +
|
||
fft_out[k + 2] * filters.data[j * n_fft + k + 2] +
|
||
fft_out[k + 3] * filters.data[j * n_fft + k + 3];
|
||
}
|
||
|
||
// handle n_fft remainder
|
||
for (; k < n_fft; k++) {
|
||
sum += fft_out[k] * filters.data[j * n_fft + k];
|
||
}
|
||
|
||
sum = log10(std::max(sum, 1e-10));
|
||
|
||
mel.data[j * mel.n_len + i] = sum;
|
||
}
|
||
}
|
||
|
||
// Otherwise fft_out are all zero
|
||
double sum = log10(1e-10);
|
||
for (; i < mel.n_len; i += n_threads) {
|
||
for (int j = 0; j < mel.n_mel; j++) {
|
||
mel.data[j * mel.n_len + i] = sum;
|
||
}
|
||
}
|
||
}
|
||
|
||
static bool log_mel_spectrogram(
|
||
whisper_state & wstate,
|
||
const float * samples,
|
||
const int n_samples,
|
||
const int /*sample_rate*/,
|
||
const int frame_size,
|
||
const int frame_step,
|
||
const int n_mel,
|
||
const int n_threads,
|
||
whisper_filters & filters,
|
||
const bool debug,
|
||
whisper_mel & mel) {
|
||
// Hanning window (Use cosf to eliminate difference)
|
||
// ref: https://pytorch.org/docs/stable/generated/torch.hann_window.html
|
||
// ref: https://github.com/openai/whisper/blob/main/whisper/audio.py#L147
|
||
fill_sin_cos_table();
|
||
|
||
// auto & filters = filters;
|
||
filters.data.resize(filters.n_mel*filters.n_fft);
|
||
auto fin = std::ifstream("./data_bin/data.bin", std::ios::binary);
|
||
if (!fin)
|
||
{
|
||
fprintf(stderr, "%s : fail to open '%s'\n", __func__, "./data_bin/data.bin");
|
||
}
|
||
fin.read((char *)filters.data.data(), filters.data.size()*sizeof(float));
|
||
fin.eof();
|
||
fin.close();
|
||
|
||
|
||
std::vector<float> hann;
|
||
hann_window(frame_size, true, hann);
|
||
|
||
// Calculate the length of padding
|
||
int64_t stage_1_pad = WHISPER_SAMPLE_RATE * 30;
|
||
int64_t stage_2_pad = frame_size / 2;
|
||
|
||
// Initialize a vector and copy data from C array to it.
|
||
std::vector<float> samples_padded;
|
||
samples_padded.resize(n_samples + stage_1_pad + stage_2_pad * 2);
|
||
std::copy(samples, samples + n_samples, samples_padded.begin() + stage_2_pad);
|
||
|
||
// pad 30 seconds of zeros at the end of audio (480,000 samples) + reflective pad 200 samples at the end of audio
|
||
std::fill(samples_padded.begin() + n_samples + stage_2_pad, samples_padded.begin() + n_samples + stage_1_pad + 2 * stage_2_pad, 0);
|
||
|
||
// reflective pad 200 samples at the beginning of audio
|
||
std::reverse_copy(samples + 1, samples + 1 + stage_2_pad, samples_padded.begin());
|
||
|
||
mel.n_mel = n_mel;
|
||
// https://github.com/pytorch/pytorch/blob/main/aten/src/ATen/native/SpectralOps.cpp#L936
|
||
// Calculate number of frames + remove the last frame
|
||
mel.n_len = (samples_padded.size() - frame_size) / frame_step;
|
||
// Calculate semi-padded sample length to ensure compatibility
|
||
mel.n_len_org = 1 + (n_samples + stage_2_pad - frame_size) / frame_step;
|
||
|
||
mel.data.resize(mel.n_mel * mel.n_len);
|
||
|
||
{
|
||
std::vector<std::thread> workers(n_threads - 1);
|
||
for (int iw = 0; iw < n_threads - 1; ++iw) {
|
||
workers[iw] = std::thread(
|
||
log_mel_spectrogram_worker_thread, iw + 1, std::cref(hann), samples_padded,
|
||
n_samples + stage_2_pad, frame_size, frame_step, n_threads,
|
||
std::cref(filters), std::ref(mel));
|
||
}
|
||
|
||
// main thread
|
||
log_mel_spectrogram_worker_thread(0, hann, samples_padded, n_samples + stage_2_pad, frame_size, frame_step, n_threads, filters, mel);
|
||
|
||
for (int iw = 0; iw < n_threads - 1; ++iw) {
|
||
workers[iw].join();
|
||
}
|
||
}
|
||
|
||
// clamping and normalization
|
||
double mmax = -1e20;
|
||
for (int i = 0; i < mel.n_mel*mel.n_len; i++) {
|
||
if (mel.data[i] > mmax) {
|
||
mmax = mel.data[i];
|
||
}
|
||
}
|
||
|
||
mmax -= 8.0;
|
||
|
||
for (int i = 0; i < mel.n_mel*mel.n_len; i++) {
|
||
if (mel.data[i] < mmax) {
|
||
mel.data[i] = mmax;
|
||
}
|
||
|
||
mel.data[i] = (mel.data[i] + 4.0)/4.0;
|
||
}
|
||
|
||
return true;
|
||
}
|
||
|
||
int whisper_pcm_to_mel_with_state(struct whisper_context * ctx, struct whisper_state * state, const float * samples, int n_samples, int n_threads) {
|
||
if (!log_mel_spectrogram(*state, samples, n_samples, WHISPER_SAMPLE_RATE, WHISPER_N_FFT, WHISPER_HOP_LENGTH, 80, n_threads, ctx->model.filters, true, state->mel)) {
|
||
printf("%s: failed to compute mel spectrogram\n", __func__);
|
||
return -1;
|
||
}
|
||
|
||
return 0;
|
||
}
|
||
|
||
static std::vector<whisper_vocab::id> tokenize(const whisper_vocab & vocab, const std::string & text) {
|
||
std::vector<std::string> words;
|
||
|
||
// first split the text into words
|
||
{
|
||
std::string str = text;
|
||
std::string pat = R"('s|'t|'re|'ve|'m|'ll|'d| ?[[:alpha:]]+| ?[[:digit:]]+| ?[^\s[:alpha:][:digit:]]+|\s+(?!\S)|\s+)";
|
||
|
||
std::regex re(pat);
|
||
std::smatch m;
|
||
|
||
while (std::regex_search(str, m, re)) {
|
||
for (auto x : m) {
|
||
words.push_back(x);
|
||
}
|
||
str = m.suffix();
|
||
}
|
||
}
|
||
|
||
// find the longest tokens that form the words:
|
||
std::vector<whisper_vocab::id> tokens;
|
||
for (const auto & word : words) {
|
||
if (word.empty()) continue;
|
||
|
||
int i = 0;
|
||
int n = word.size();
|
||
while (i < n) {
|
||
int j = n;
|
||
bool found = false;
|
||
while (j > i) {
|
||
auto sub = word.substr(i, j-i);
|
||
auto it = vocab.token_to_id.find(sub);
|
||
if (it != vocab.token_to_id.end()) {
|
||
tokens.push_back(it->second);
|
||
i = j;
|
||
found = true;
|
||
break;
|
||
}
|
||
--j;
|
||
}
|
||
if (!found) {
|
||
printf("unknown token\n");
|
||
++i;
|
||
}
|
||
}
|
||
}
|
||
|
||
return tokens;
|
||
}
|