singe/util/makeMenuSound.py

# The menu backdrop's sound:  the sting the intro runs to, with a bed underneath it that fades out
# as the menu takes the screen.  The three things the picture shows -- the charge going off, the
# logo bursting out of it, the dragon rearing and breathing fire -- are recordings, three sounds
# from soundbible.com under the Creative Commons Attribution 3.0 licence, kept as FLAC under
# assets/samples and credited in LICENSES:  Mark DiAngelo's "Explosion Ultra Bass", Mike Koenig's
# "Large Fireball", and Daniel Simon's "Dragon Fire Breath and Roar".  They replaced the blast and
# the flame this script used to synthesise, which never sounded like the things themselves.  What
# is between them -- the wingbeats, the riser as the grid comes up, the logo leaving, and the bed
# -- is still made here from oscillators and noise, in a room that is a convolution, and numpy does
# the arithmetic as it does for the model tools.
#
# One file comes out of it, menuIntro.flac, exactly as long as the backdrop's intro.  Nothing loops:
# music under a menu that is waiting for someone to choose a game wears out its welcome in about
# fifteen seconds, so the bed fades away once the menu has the screen and what is left is quiet.
# That is also why the recording's looping section carries silence.
#
# No length is written down here.  util/renderMenuVideo.py asks the backdrop where its intro ends
# and passes it in, so the sound cannot drift out of step with the picture.
#
# Usage:  python3 util/renderMenuVideo.py        (which calls makeSound below)
#         python3 util/makeMenuSound.py --intro 9.90 --out assets
import argparse
import math
import os
import subprocess
import wave
import numpy as np

RATE = 44100
SEED = 20260913            # Two runs write the same file, so a re-render can be compared with the
                           # last one and the difference is the picture, not the dice.
TAIL = 0.006               # Seconds an envelope is given to reach silence before it is cut off.
BED_PEAK = 0.34            # How loud the bed is.  Everything else is fitted around it, because it
                           # is the one thing playing under the whole intro.
PEAK = 0.97                # Where the limiter tops out.
KNEE = 0.80                # and where it starts to bend.  Below this nothing is touched at all.
                           # High, so the charge's transient comes through as it was made rather
                           # than rounded off:  the limiter is here to catch the last of it, not to
                           # flatten the loudest moment in the file into the same shape as the rest.

# The intro's marks, in seconds, from assets/Backdrop.singe.  They are here rather than read out of
# it because the Lua is the picture's copy and this is the sound's;  what has to agree between the
# two -- where the whole thing ends -- is passed in instead of guessed at.
FLY_START = 0.30
FLY_END = 1.90
FLAP_FAST = 13.0
FLAP_SLOW = 3.4
BREATH_START = 3.46
BREATH_END = 4.35
REAR_DONE = 4.90           # The head is back where it started;  the dragon's fire has gone by here.
GRID_UP = 4.30
GRID_LIT = 5.60
LIFT_FROM = 5.90
LIFT_TO = 7.30             # The menu has the screen from here, and the music starts leaving.

# The tempo comes out of the picture:  four bars land exactly on LIFT_TO, so the last chord of the
# phrase is the one the menu arrives on.
BEATS = 4
BARS_TO_MENU = 4
ROOT = 220.0               # A3.  The grid is magenta and the sun is orange;  the key is A minor.
HARMONICS = 12             # Partials in the sawtooth.  Twelve keeps the top of the arpeggio, the
                           # highest note here, inside half the sample rate, so nothing aliases.

# The chord for each bar in turn, as semitones from the root, and the bass note beneath it.
CHORDS = (((0, 3, 7, 12), -12), ((-4, 0, 5, 8), -16), ((-5, 0, 3, 7), -17), ((-4, 0, 5, 8), -16))
ARPEGGIO = (0, 7, 12, 15, 12, 7, 12, 3)   # Sixteenths, the same shape over every chord.

# The room the sting is heard in.  A blast with nothing around it is a click:  what makes it big is
# the second and a half of room that answers it.
ROOM_SECONDS = 2.6
ROOM_DECAY = 0.85          # Seconds the room falls by a factor of e.
ROOM_DARK = 1500           # It loses its top as it goes, the way a room does, and a long way off
                           # only the bottom of a blast is left at all.
ROOM_PREDELAY = 0.014      # The dry sound is heard on its own first, or it arrives already blurred.
SAMPLES = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "assets", "samples")
EXPLOSION_LEVEL = 1.0      # The recordings against each other:  the charge fills the scale, the
DRAGON_LEVEL = 1.2         # dragon is driven into the limiter over it, the fireball is a rush
FIREBALL_LEVEL = 0.6       # rather than a blow.
EXPLOSION_DRIVE = 1.6      # The explosion through a soft clipper, so its body is as loud as its
                           # crack and it hits rather than pops.
STING_PEAK = 1.3           # The sting above full scale before the limiter:  the limiter bends
                           # everything over its knee towards the ceiling, so this is the loudness
                           # -- the punch -- of the whole intro.  It was 1.15, and polite;  at 2.2
                           # the whole first four seconds were one flat wall.
EXPLOSION_HOLD = 0.4       # Seconds the explosion's rumble runs at full before it is let go,
EXPLOSION_GONE = 2.0       # and where it has gone:  the roar is heard over silence, not rumble.
                           # (At 0.8 and 3.2 the rumble buried the roar altogether.)
DRAGON_FIRE_AT = 2.75      # Seconds into the dragon recording where its roar turns to fire;  the
                           # recording is laid so that moment is the animation's BREATH_START.
DRAGON_ROAR_GAIN = 2.0     # The roar starts small in that recording and builds;  its start is
                           # brought up so it is heard as the dragon flies in, not only as it rears.
DRAGON_FIRE_GAIN = 3.0     # The fire sits some eight decibels under the roar before it, and it is
                           # the fire the picture shows;  brought up past it.
ROOM_SLAPS = ((0.061, 0.30), (0.113, 0.20), (0.187, 0.12))   # Distinct returns off whatever is out
                           # there.  A blast in the open is heard once and then answered; a smooth
                           # tail on its own is a plate reverb, which is a studio and not a place.


def seconds(count):
	return np.arange(count) / RATE


def note(semitones):
	return ROOT * (2.0 ** (semitones / 12.0))


# Convolution through the FFT.  Every fixed filter here is one of these -- a one pole low pass is
# just a decaying exponential to convolve with -- which keeps the whole file to array arithmetic
# instead of a per sample loop, and makes the reverb affordable at all.
def convolve(signal, kernel):
	size = 1
	while size < len(signal) + len(kernel):
		size *= 2
	out = np.fft.irfft(np.fft.rfft(signal, size) * np.fft.rfft(kernel, size), size)

	return out[:len(signal)]


# A one pole low pass, as the exponential it is.  The kernel is cut off where it has fallen below
# a hundred thousandth, which is inaudible and keeps the transform small.
def lowpass(signal, cutoff):
	pole = math.exp(-2.0 * math.pi * cutoff / RATE)
	length = min(int(math.log(1e-5) / math.log(pole)) + 1, len(signal))
	kernel = (1.0 - pole) * pole ** np.arange(length)

	return convolve(signal, kernel)


def highpass(signal, cutoff):
	return signal - lowpass(signal, cutoff)


def bandpass(signal, low, high):
	return highpass(lowpass(signal, high), low)


# A two pole state variable filter with a moving cutoff, which rings at the cutoff as the resonance
# goes up.  The ring is the whole point of a riser:  a swept resonance is what makes noise sound
# like it is climbing towards something.
def resonant(signal, cutoffs, resonance):
	out = np.empty(len(signal))
	steps = 2.0 * np.sin(np.pi * np.clip(cutoffs, 10.0, RATE / 2.2) / RATE)
	damping = 1.0 / resonance
	low = 0.0
	band = 0.0

	for i, step in enumerate(steps):
		high = signal[i] - low - band * damping
		band += step * high
		low += step * band
		out[i] = low

	return out


# An envelope follower:  fast to rise, slow to fall, which is how a compressor hears a sound and
# how the blast gets to push everything else out of its way.
def follow(signal, attack, release):
	out = np.empty(len(signal))
	rise = math.exp(-1.0 / (attack * RATE))
	fall = math.exp(-1.0 / (release * RATE))
	last = 0.0

	for i, value in enumerate(np.abs(signal)):
		pole = rise if value > last else fall
		last = value * (1.0 - pole) + last * pole
		out[i] = last

	return out


# Everything that is not the blast, pushed out of the blast's way and let back in.  Punch is
# contrast:  a loud sound with nothing standing next to it is heard as louder than the same sound
# with the rest of the mix holding its level underneath.
def duck(signal, trigger, amount, attack, release):
	envelope = follow(trigger, attack, release)
	worst = envelope.max()

	return signal * (1.0 - amount * envelope / worst) if worst > 0.0 else signal


# The room, as an impulse to convolve with:  noise that dies away, darkening as it goes, with the
# first few milliseconds left empty so the dry sound arrives before its reflections do.
def room(rng):
	count = int(ROOM_SECONDS * RATE)
	impulse = rng.standard_normal(count) * np.exp(-seconds(count) / ROOM_DECAY)
	for at, level in ROOM_SLAPS:
		impulse[int(at * RATE)] += level
	impulse = lowpass(impulse, ROOM_DARK)
	impulse[:int(ROOM_PREDELAY * RATE)] = 0.0

	return impulse / math.sqrt(float(np.sum(impulse * impulse)))


# An oscillator whose frequency is given a sample at a time.
def sweep(frequencies):
	return np.sin(2.0 * math.pi * np.cumsum(frequencies) / RATE)


# A band limited sawtooth, built one harmonic at a time.
def saw(t, frequency):
	out = np.zeros(len(t))
	for h in range(1, HARMONICS + 1):
		if frequency * h < RATE / 2.0:
			out += np.sin(2.0 * math.pi * frequency * h * t) / h

	return out * 2.0 / math.pi


def noise(count, rng):
	return rng.standard_normal(count)


# An envelope that rises in attack seconds and falls away over decay, fast to start and slow to
# finish:  a struck sound rather than a triangle.  An exponential never actually reaches zero, so
# the last few milliseconds are taken down by hand;  cutting one off where it still had a tenth of
# its level left is a click, and the bed has hundreds of these in it.
def hit(attack, decay, length):
	count = int(length * RATE)
	t = seconds(count)
	rise = np.clip(t / max(attack, 1e-6), 0.0, 1.0)
	fall = np.exp(-np.clip(t - attack, 0.0, None) / decay)
	close = np.clip((count - 1 - np.arange(count)) / (TAIL * RATE), 0.0, 1.0)

	return rise * fall * close


# A window that comes up, holds, and goes down again:  for the parts of the intro that have a
# length of their own rather than a decay.
def swell(length, rise, fall):
	count = int(length * RATE)
	up = np.clip(np.arange(count) / (rise * RATE), 0.0, 1.0)
	down = np.clip((count - 1 - np.arange(count)) / (fall * RATE), 0.0, 1.0)

	return np.minimum(up, down)


# Lays a signal into the take at a time, as long as whatever is shorter.
def lay(into, signal, start, level=1.0):
	first = max(int(start * RATE), 0)
	length = min(len(signal), len(into) - first)
	if length > 0:
		into[first:first + length] += signal[:length] * level


# Where the dragon's wings reach the bottom of a beat, worked out the way assets/Backdrop.singe
# works it out:  the flap is sin(t * rate) with the rate easing from fast to slow as the logo flies
# out of the blast, so the gusts land on the animation rather than near it.
def flapTimes(until):
	t = np.arange(0.0, until, 1.0 / 240.0)
	span = np.clip((t - FLY_START) / (FLY_END - FLY_START), 0.0, 1.0)
	value = np.sin(t * (FLAP_FAST + (FLAP_SLOW - FLAP_FAST) * (1.0 - (1.0 - span) ** 3)))
	falling = (value[:-1] > 0.0) & (value[1:] <= 0.0) & (t[:-1] > FLY_START)

	return t[:-1][falling]


# A recording, decoded to this script's rate as one channel and brought to full scale, so what is
# laid into the sting is the sound and the level is the level asked for.
def sample(name):
	path = os.path.join(SAMPLES, name + ".flac")
	raw = subprocess.run(["ffmpeg", "-loglevel", "error", "-i", path, "-f", "f32le", "-ac", "1", "-ar", str(RATE), "-"], check=True, capture_output=True).stdout

	return normalise(np.frombuffer(raw, dtype="<f4").astype(np.float64), 1.0)


# A recording let go:  as it is until `hold` seconds, then away to nothing by `gone`, and cut there.
def letGo(signal, hold, gone):
	return fade(signal[:int(gone * RATE)].copy(), hold)


# The sting:  the recordings where the picture has something to show, what is made here between
# them in a room, and everything getting out of the charge's way.  The bed is not in here;  it goes
# underneath afterwards.
def stingTake(length, rng):
	count = int(length * RATE)
	made = np.zeros(count)
	heard = np.zeros(count)
	charge = np.zeros(count)
	dragonAt = BREATH_START - DRAGON_FIRE_AT

	# The charge:  the explosion from its first sample, its rumble let go over the seconds after,
	# so the dragon is heard over what is left of it rather than through it.
	blast = np.tanh(sample("explosionUltraBass") * EXPLOSION_DRIVE) / math.tanh(EXPLOSION_DRIVE)
	lay(charge, letGo(blast, EXPLOSION_HOLD, EXPLOSION_GONE), 0.0, EXPLOSION_LEVEL)

	# The logo coming out of the blast:  the fireball, leaving as the logo does.
	lay(heard, sample("largeFireball"), FLY_START, FIREBALL_LEVEL)

	# The dragon:  the roar as it flies in and rears, the fire on the breath.  The recording is
	# laid so the moment its roar turns to fire is the moment the animation's does, and its fire
	# is let go as the head comes back.
	dragon = sample("dragonFireBreathAndRoar")
	dragonT = seconds(len(dragon))
	# The roar's gain eases from its start to nothing extra by the fire;  the fire's steps in there.
	dragon *= 1.0 + (DRAGON_ROAR_GAIN - 1.0) * np.clip(1.0 - dragonT / DRAGON_FIRE_AT, 0.0, 1.0)
	dragon *= 1.0 + (DRAGON_FIRE_GAIN - 1.0) * np.clip((dragonT - DRAGON_FIRE_AT) / 0.05, 0.0, 1.0)
	lay(heard, letGo(dragon, BREATH_END - dragonAt, REAR_DONE - dragonAt), dragonAt, DRAGON_LEVEL)

	# Wings.
	gust = bandpass(noise(int(0.35 * RATE), rng), 180, 2200) * hit(0.03, 0.12, 0.35)
	for at in flapTimes(length):
		lay(made, gust, at, 0.16)

	# The grid coming up out of the dark, twice over:  a note that rises with it, and noise through
	# a resonance climbing the same way, which is the sound of something being switched on.
	climb = GRID_LIT - GRID_UP
	count = int(climb * 1.1 * RATE)
	ramp = np.clip(seconds(count) / climb, 0.0, 1.0)
	lay(made, sweep(note(-12) * (1.0 + ramp * 3.0)) * swell(climb * 1.1, 0.9, 0.5), GRID_UP, 0.11)
	lay(made, resonant(noise(count, rng), 180.0 * (1.0 + ramp * 32.0), 9.0) * swell(climb * 1.1, 1.0, 0.35), GRID_UP, 0.11)

	# The logo leaving.
	lay(made, bandpass(noise(int((LIFT_TO - LIFT_FROM) * RATE), rng), 2000, 11000) * swell(LIFT_TO - LIFT_FROM, 0.5, 0.6), LIFT_FROM, 0.08)

	# What was made here gets out of the charge's way and comes back over the next quarter second;
	# the recordings do not, since the duck follows the whole of the explosion's rumble and took
	# the roar down with it.  The room is for what was made here;  the recordings carry their own.
	made = duck(made, charge, 0.8, 0.003, 0.22)
	space = room(rng)

	return made + convolve(made, space) * 0.42 + heard + charge


# The bed:  a pad, a bass and an arpeggio, bar after bar for as long as it is wanted.  It is played
# straight through rather than made once and repeated, so the echo and the pad carry across a bar
# line the way they would if somebody played it.
def bedTake(length, beat, rng):
	count = int(length * RATE)
	out = np.zeros(count)
	sixteenth = beat / 4.0
	held = swell(BEATS * beat + 0.25, 0.35, 0.5)
	plucked = hit(0.004, sixteenth * 1.6, sixteenth * 4)
	struck = hit(0.01, beat * 0.7, beat * 2)
	heldT = seconds(len(held))
	pluckT = seconds(len(plucked))
	struckT = seconds(len(struck))

	for bar in range(int(math.ceil(length / (BEATS * beat)))):
		chord, bass = CHORDS[bar % len(CHORDS)]
		barAt = bar * BEATS * beat

		# The pad:  the chord held for the whole bar, soft, slow to arrive, and detuned against
		# itself so it moves rather than sits there.
		for semitone in chord:
			lay(out, (saw(heldT, note(semitone)) + saw(heldT, note(semitone) * 1.004)) * held, barAt, 0.045)
		# The bass:  one note a bar, and another on the last beat to lean into the next one.
		for at, level in ((barAt, 0.5), (barAt + 3 * beat, 0.34)):
			lay(out, np.sin(2.0 * math.pi * note(bass) * struckT) * struck, at, level)
			lay(out, saw(struckT, note(bass) * 2) * struck, at, level * 0.25)
		# The arpeggio:  sixteenths, plucked, riding on top.
		for step in range(BEATS * 4):
			semitone = chord[0] + ARPEGGIO[step % len(ARPEGGIO)]
			lay(out, saw(pluckT, note(semitone) * 2) * plucked, barAt + step * sixteenth, 0.06)

	# A quarter note echo, and a breath of air under all of it.
	echo = int(round(beat * RATE))
	for _ in range(3):
		out[echo:] += out[:-echo] * 0.32
	out += lowpass(noise(count, rng), 900) * 0.012

	return out


def normalise(signal, peak):
	worst = float(np.max(np.abs(signal)))

	return signal * (peak / worst) if worst > 0.0 else signal


# The music leaves as the menu arrives:  full level until LIFT_TO, then away to nothing by the end
# of the file.  A cosine rather than a straight line, because a straight fade is heard as a shove
# at the start and a long nothing at the end.
def fade(signal, at):
	first = int(at * RATE)
	away = (1.0 + np.cos(math.pi * np.arange(len(signal) - first) / (len(signal) - first))) / 2.0
	signal[first:] *= away

	return signal


# A soft limiter, so the loudest moment can be loud without deciding how loud everything else is.
# Below the knee nothing is touched;  above it the curve bends over towards the ceiling, which is
# what lets the charge sit at the top of the scale without the rest of the intro being scaled down
# to make room for its peak.
def limit(signal):
	over = np.abs(signal) > KNEE
	room = PEAK - KNEE
	signal = signal.copy()
	signal[over] = np.sign(signal[over]) * (KNEE + room * np.tanh((np.abs(signal[over]) - KNEE) / room))

	return signal


# A little width:  the same sound a few samples apart is enough for a menu, and it stays mono
# compatible, which matters on a cabinet with one speaker.
def stereo(signal, shift=48):
	late = np.concatenate((np.zeros(shift), signal[:-shift]))

	return np.stack((signal * 0.92 + late * 0.08, signal * 0.92 - late * 0.08), axis=1)


# FLAC rather than a compressed format:  it is what the video's audio track is muxed from, and a
# lossy encoder pads both ends of what it encodes, which the engine's seeks would find.
def writeFlac(path, frames):
	temp = path + ".wav"
	data = np.clip(frames, -1.0, 1.0)
	with wave.open(temp, "wb") as out:
		out.setnchannels(2)
		out.setsampwidth(2)
		out.setframerate(RATE)
		out.writeframes((data * 32767.0).astype("<i2").tobytes())
	subprocess.run(["ffmpeg", "-y", "-loglevel", "error", "-i", temp, "-c:a", "flac", path], check=True)
	os.unlink(temp)


# Writes the file and answers its path.  Called by util/renderMenuVideo.py with the length the
# backdrop itself reported, so nothing here has to know how long the intro is.
def makeSound(introSeconds, folder):
	rng = np.random.default_rng(SEED)
	path = os.path.join(folder, "menuIntro.flac")
	beat = LIFT_TO / (BARS_TO_MENU * BEATS)
	bed = normalise(bedTake(introSeconds, beat, rng), BED_PEAK)
	# Not normalised:  every recording is already at full scale and laid at its level, so the
	# blast keeps the ceiling whatever the dragon is driven to.  (Normalising the sum handed the
	# scale to whichever was loudest, and the last turn of the dragon's level took the blast down
	# with it.)
	sting = stingTake(introSeconds, rng) * STING_PEAK

	# The bed comes up under the flame and is established by the time the grid is.
	rising = np.clip((seconds(len(bed)) - BREATH_START) / (GRID_UP - BREATH_START), 0.0, 1.0)
	writeFlac(path, stereo(fade(limit(sting + bed * rising), LIFT_TO)))

	return path


def main():
	parser = argparse.ArgumentParser(description="write the menu backdrop's sound")
	parser.add_argument("--intro", type=float, required=True, help="seconds the backdrop's intro lasts")
	parser.add_argument("--out", default=os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "assets"))
	args = parser.parse_args()
	print(makeSound(args.intro, args.out))


if __name__ == "__main__":
	main()