Use colors to mix different audio recordings

Convert colors to OSC messages

The footage from the ISS is from the year 2015. 4 commercial HDTV cameras were in operation there. There was then a few years of video stream from the ISS that everyone could view or record.

The sound comes from numbers stations of various intelligence agencies. These recordings were collected by www.numbers-stations.com .

The graphic animation is permanently sampled in 4 moving areas. With the color values of these areas the announcements of the numbers stations are continuously overlaid.

Where to scan the image.

Per frame, the x-positions at which the image is scanned are recalculated by Perlin noise functions. The size of the scanned areas are calculated by noise function too.

Four areas are sampled at the same time. For each area hue, saturation and brightness is calculated. So 12 separate values can be used to mix the 6 prerecorded audio files.

(ns lv.demo)
(use 'lv.core)

(layout "grid" [img0 img1]
               [ras2 ras3])
(image "iss/scale_100.png")

(texpr
  ; use nanoKONTROL2 als input
  ; ks0    0.82677
  ; ks1    0.49606
  ; ks2    0.30709
  ; ks3    0.12598
  (t0 (- 1 (* 2 ks0)))
  (t1 (- 1 (* 2 ks1)))
  (t2 (- 1 (* 2 ks2)))
  (t3 (- 1 (* 2 ks3)))
)

(rgb :img0 [c0]
  (ty (* 19.4 (unoise 0.1 (* 0.001 f))))
  (tx -0.08)
  (sx 0.35)
  (rgb c0.rgb)
)

(rgb :img1 [c0]
  (rot (* -0.005 f))
  (ty (* 0.094 19.4))
  (sx 2)
  (rgb c0.rgb)
)

(rgb :ras2 [img0 img1]
  (rgb (mix (sqr img0.rgb) img1.rgb 0.24))
)
(render 2)

(inspect [ras2]
  "color2sound/analyse_quadhead_color.cl"
  (channels 4)
  (x0 (* 1.5 (snoise (* 0.0051 f) 0.1 0.2)))
  (y0 t0)
  (r0 (* 0.25 (unoise f 0.3 0.4)))

  (x1 (* 1.5 (snoise (* 0.0053 f) 0.5 0.6)))
  (y1 t1)
  (r1 (* 0.15 (unoise (* 0.1 f) 0.7 0.8)))

  (x2 (* 1.5 (snoise (* 0.0047 f) 0.5 0.6)))
  (y2 t2)
  (r2 (* 0.15 (unoise (* 0.157 f) 0.9 0.1)))

  (x3 (* 1.5 (snoise (* 0.0043 f) 0.5 0.6)))
  (y3 t3)
  (r3 (* 0.15 (unoise (* 0.071 f) 0.2 0.5)))
)

Convert OSC messages to sound

This is a script for SuperCollider. The SynthDef AudioMix is instantiated twice. On the left and the right audio channel the same synth is running, but they are supplied with different audio recordings and different the values from sampling areas.

s.waitForBoot({

  ~b1 = Buffer.read(s, "~/soundscape/numbers-stations/E05_Cynthia_msg-mono.wav");
  ~b2 = Buffer.read(s, "~/soundscape/numbers-stations/G11_RW_Observer-mono.wav");
  ~b3 = Buffer.read(s, "~/soundscape/numbers-stations/G14_Conet_Project-part_music-mono.wav");
  ~b4 = Buffer.read(s, "~/soundscape/numbers-stations/G14_Conet_Project-part_numbers-mono.wav");
  ~b5 = Buffer.read(s, "~/soundscape/numbers-stations/V15-2017-2704-1615z-mono.wav");
  ~b6 = Buffer.read(s, "~/soundscape/numbers-stations/V30_Token-mono.wav");

  // merge prerecorded sound files
  SynthDef(\AudioMix,
    { | out = 0,
        bufnum1,
        bufnum2,
        bufnum3,
        bufnum4,
        hue1=0.5,
        saturation1=0.5,
        brightness1=0.5,
        hue2=0.5,
        saturation2=0.5,
        brightness2=0.5 |
      var snd1 = hue1*PlayBuf.ar(1, bufnum1, BufRateScale.kr(bufnum1), loop:1);
      var snd2 = saturation1*PlayBuf.ar(1, bufnum2, BufRateScale.kr(bufnum2), loop:1);
      var chainA = FFT(LocalBuf(2048), snd1);
      var chainB = FFT(LocalBuf(2048), snd2);
      var chainC = PV_XFade(chainA, chainB, brightness1);
      var snd3 = IFFT(chainC);

      var snd4 = hue2*PlayBuf.ar(1, bufnum3, BufRateScale.kr(bufnum3), loop:1);
      var snd5 = saturation2*PlayBuf.ar(1, bufnum4, BufRateScale.kr(bufnum4), loop:1);
      var chainD = FFT(LocalBuf(2048), snd4);
      var chainE = FFT(LocalBuf(2048), snd5);
      var chainF = PV_XFade(chainD, chainE, brightness2);
      var snd6 = IFFT(chainF);

      Out.ar(out, Mix.new([snd3, snd6])*0.5);
  }, \kr!11).add;

  0.5.wait;
  ~left  = Synth.new(\AudioMix,
                     [ \out, 0,
                       \bufnum1, ~b1,
                       \bufnum2, ~b2,
                       \bufnum3, ~b3,
                       \bufnum4, ~b4]);
  ~right = Synth.new(\AudioMix,
                     [ \out, 1,
                       \bufnum1, ~b3,
                       \bufnum2, ~b4,
                       \bufnum3, ~b5,
                       \bufnum4, ~b6]);

  h.free;
  h = OSCFunc({ |msg, time, addr, recvPort|
    // from area x0
    ~left.set(\hue1, msg[5]);
    ~left.set(\saturation1, msg[6]);
    ~left.set(\brightness1, msg[7]);
    // from area x1
    ~left.set(\hue2, msg[11]);
    ~left.set(\saturation2, msg[12]);
    ~left.set(\brightness2, msg[13]);

    // from area x2
    ~right.set(\hue1, msg[17]);
    ~right.set(\saturation1, msg[18]);
    ~right.set(\brightness1, msg[19]);
    // from area x3
    ~right.set(\hue2, msg[23]);
    ~right.set(\saturation2, msg[24]);
    ~left.set(\brightness2, msg[25]);
  }, '/f_hsb4c');

  q.free;
  q = OSCFunc({ |msg, time, addr, recvPort|
    // stop
    [time, msg].postln;
    s.quit;
    0.exit;
  }, '/f_collider/quit');

});