Signalsmith-Audio/signalsmith-stretch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Ensure previous input spectrum is always one interval behind.

Browse Source 
This increases CPU when time-stretching, but improves tuning.
This commit is contained in:
Geraint 2022-12-04 09:51:58 +00:00
parent bb1bc7a6c4
commit 6364651d1b
1 changed files with 47 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
signalsmith-stretch.h
View File

@ -47,17 +47,15 @@ struct SignalsmithStretch {
void configure(int nChannels, int blockSamples, int intervalSamples) { void configure(int nChannels, int blockSamples, int intervalSamples) {
channels = nChannels; channels = nChannels;
stft.resize(channels, blockSamples, intervalSamples); stft.resize(channels, blockSamples, intervalSamples);
inputBuffer.resize(channels, blockSamples); inputBuffer.resize(channels, blockSamples + intervalSamples + 1);
timeBuffer.assign(stft.fftSize(), 0); timeBuffer.assign(stft.fftSize(), 0);
channelBands.assign(stft.bands()*channels, Band()); channelBands.assign(stft.bands()*channels, Band());
// Various phase rotations // Various phase rotations
rotCentreSpectrum.resize(stft.bands()); rotCentreSpectrum.resize(stft.bands());
rotPrevInput.assign(stft.bands(), 0); rotPrevInterval.assign(stft.bands(), 0);
rotPrevInputShift = -1;
rotPrevOutput.resize(stft.bands());
timeShiftPhases(blockSamples*Sample(-0.5), rotCentreSpectrum); timeShiftPhases(blockSamples*Sample(-0.5), rotCentreSpectrum);
timeShiftPhases(-intervalSamples, rotPrevOutput); timeShiftPhases(-intervalSamples, rotPrevInterval);
peaks.reserve(stft.bands()); peaks.reserve(stft.bands());
energy.resize(stft.bands()); energy.resize(stft.bands());
smoothedEnergy.resize(stft.bands()); smoothedEnergy.resize(stft.bands());
@ -129,11 +127,8 @@ struct SignalsmithStretch {
int inputInterval = inputOffset - prevInputOffset; int inputInterval = inputOffset - prevInputOffset;
prevInputOffset = inputOffset; prevInputOffset = inputOffset;
if (inputInterval > 0) { bool newSpectrum = (inputInterval > 0);
if (inputInterval != rotPrevInputShift) { // Only recompute if needed if (newSpectrum) {
timeShiftPhases(-inputInterval, rotPrevInput);
rotPrevInputShift = inputInterval;
}
for (int c = 0; c < channels; ++c) { for (int c = 0; c < channels; ++c) {
// Copy from the history buffer, if needed // Copy from the history buffer, if needed
auto &&bufferChannel = inputBuffer[c]; auto &&bufferChannel = inputBuffer[c];
@ -147,17 +142,42 @@ struct SignalsmithStretch {
} }
stft.analyse(c, timeBuffer); stft.analyse(c, timeBuffer);
} }
}
for (int c = 0; c < channels; ++c) { for (int c = 0; c < channels; ++c) {
auto bands = bandsForChannel(c); auto bands = bandsForChannel(c);
auto &&spectrumBands = stft.spectrum[c]; auto &&spectrumBands = stft.spectrum[c];
for (int b = 0; b < stft.bands(); ++b) { for (int b = 0; b < stft.bands(); ++b) {
bands[b].input = signalsmith::perf::mul(spectrumBands[b], rotCentreSpectrum[b]); bands[b].input = signalsmith::perf::mul(spectrumBands[b], rotCentreSpectrum[b]);
}
}
if (inputInterval != stft.interval()) { // make sure the previous input is the correct distance in the past
int prevIntervalOffset = inputOffset - stft.interval();
for (int c = 0; c < channels; ++c) {
// Copy from the history buffer, if needed
auto &&bufferChannel = inputBuffer[c];
for (int i = 0; i < std::min(-prevIntervalOffset, stft.windowSize()); ++i) {
timeBuffer[i] = bufferChannel[i + prevIntervalOffset];
}
// Copy the rest from the input
auto &&inputChannel = inputs[c];
for (int i = std::max<int>(0, -prevIntervalOffset); i < stft.windowSize(); ++i) {
timeBuffer[i] = inputChannel[i + prevIntervalOffset];
}
stft.analyse(c, timeBuffer);
}
for (int c = 0; c < channels; ++c) {
auto bands = bandsForChannel(c);
auto &&spectrumBands = stft.spectrum[c];
for (int b = 0; b < stft.bands(); ++b) {
bands[b].prevInput = signalsmith::perf::mul(spectrumBands[b], rotCentreSpectrum[b]);
}
}
} }
} }
processSpectrum(inputInterval); Sample timeFactor = stft.interval()/std::max<Sample>(1, inputInterval);
processSpectrum(newSpectrum, timeFactor);
for (int c = 0; c < channels; ++c) { for (int c = 0; c < channels; ++c) {
auto bands = bandsForChannel(c); auto bands = bandsForChannel(c);
@ -223,8 +243,7 @@ private:
int prevInputOffset = -1; int prevInputOffset = -1;
std::vector<Sample> timeBuffer; std::vector<Sample> timeBuffer;
std::vector<Complex> rotCentreSpectrum, rotPrevOutput, rotPrevInput; std::vector<Complex> rotCentreSpectrum, rotPrevInterval;
int rotPrevInputShift = -1;
Sample bandToFreq(int b) const { Sample bandToFreq(int b) const {
return (b + Sample(0.5))/stft.fftSize(); return (b + Sample(0.5))/stft.fftSize();
} }
@ -309,20 +328,18 @@ private:
} }
std::vector<int> maxEnergyChannel; std::vector<int> maxEnergyChannel;
void processSpectrum(int inputInterval) { void processSpectrum(bool newSpectrum, Sample timeFactor) {
int outputInterval = stft.interval();
int bands = stft.bands(); int bands = stft.bands();
Sample rate = outputInterval/std::max<Sample>(1, inputInterval); timeFactor = std::min<Sample>(2, timeFactor); // For now, limit the intra-block time stretching to 2x
rate = std::min<Sample>(2, rate); // For now, limit the intra-block time stretching to 2x
if (inputInterval > 0) { if (newSpectrum) {
for (int c = 0; c < channels; ++c) { for (int c = 0; c < channels; ++c) {
auto bins = bandsForChannel(c); auto bins = bandsForChannel(c);
for (int b = 0; b < stft.bands(); ++b) { for (int b = 0; b < stft.bands(); ++b) {
auto &bin = bins[b]; auto &bin = bins[b];
bin.prevOutput = signalsmith::perf::mul(bin.prevOutput, rotPrevOutput[b]); bin.prevOutput = signalsmith::perf::mul(bin.prevOutput, rotPrevInterval[b]);
bin.prevInput = signalsmith::perf::mul(bin.prevInput, rotPrevInput[b]); bin.prevInput = signalsmith::perf::mul(bin.prevInput, rotPrevInterval[b]);
} }
} }
} }
@ -354,10 +371,10 @@ private:
prediction.freqPrediction = signalsmith::perf::mul(outputBin.prevOutput, freqTwist); prediction.freqPrediction = signalsmith::perf::mul(outputBin.prevOutput, freqTwist);
if (b > 0) { if (b > 0) {
Complex downInput = getFractional<&Band::input>(c, mapPoint.inputBin - rate); Complex downInput = getFractional<&Band::input>(c, mapPoint.inputBin - timeFactor);
prediction.shortVerticalTwist = signalsmith::perf::mul<true>(prediction.input, downInput); prediction.shortVerticalTwist = signalsmith::perf::mul<true>(prediction.input, downInput);
if (b > longVerticalStep) { if (b > longVerticalStep) {
Complex longDownInput = getFractional<&Band::input>(c, mapPoint.inputBin - longVerticalStep*rate); Complex longDownInput = getFractional<&Band::input>(c, mapPoint.inputBin - longVerticalStep*timeFactor);
prediction.longVerticalTwist = signalsmith::perf::mul<true>(prediction.input, longDownInput); prediction.longVerticalTwist = signalsmith::perf::mul<true>(prediction.input, longDownInput);
} else { } else {
prediction.longVerticalTwist = 0; prediction.longVerticalTwist = 0;
@ -439,7 +456,7 @@ private:
} }
} }
if (inputInterval > 0) { if (newSpectrum) {
for (auto &bin : channelBands) { for (auto &bin : channelBands) {
bin.prevOutput = bin.output; bin.prevOutput = bin.output;
bin.prevInput = bin.input; bin.prevInput = bin.input;