back home

Audichromos - interactive music mixer

interacting with music and visuals via color tracking

Team:
Jonas Pflug
Katja Rößler
Nico Gaupp
Michael Mack

Supervision:
Franklin Hernandez-Castro

Key Skills:
Programming
Concept Ideation
Time Management

Time Intensity:
Very High

Difficulty:
High

‍

Concept & Implementation

Colored balls are used to activate various music components or, when the ball is removed, to deactivate them.
For example, if you hold a red ball in front of the camera, a double bass will sound. This way you can change the music as you like and experiment with tone sequences. Each sphere is represented by a circle on the screen. If you move a ball, the respective circle will follow it.
On the complete screen there are needle-like lines which move away from the colored circles on the screen. By moving the ball you can create individual music and organic visuals.

Audichromos was completely implemented in Javascript with several libraries, the music components and the soundtrack in the teaser video are also composed by us.

Our concept board

‍

‍

Our Code

window.onload = onReady;

var frameCounter;
var canvas;
var ctx;

// Frame Rate
var filterStrength = 5;
var frameTime = 0,
lastLoop = new Date,
thisLoop;
var rectX, rectY, rectWidth, rectHeight;

var mouseX = 5000;
var mouseY = 5000;

var physics = new Physics(0, 1);

var myParticles = [];
var myOrgParticles = [];
var mySprings = [];

// Register other Colors to Track in Visuals
var yellowX = 50;
var yellowY = 50;

var greenX = 450;
var greenY = 50;

var redX = 50;
var redY = 300;

var blueX = 450;
var blueY = 300;

var mouseParticle;
var mouseParticle2;
var mouseParticle3;
var mouseParticle4;
var mouseParticle5;

// Grid
var xCount = 50;
var yCount = 50;

var gridSizeX, gridSizeY, gapX, gapY;
var border = 5;

////////////////////////// onReady /////////////////////////////////////////////////////////////////

function onReady() {
video = document.getElementById('video');
canvas = document.getElementById('canvas');
ctx = canvas.getContext('2d');
frameCounter = 0;
canvas.addEventListener('mousemove', pick);

////////////////////////// Color-Registration ////////////////////////////////////////////////////

tracking.ColorTracker.registerColor('Blau', function(r, g, b) {
if (r > 0 && r < 30 && g > 40 && g < 110 && b > 120 && b < 220) {
return true;
}
return false;
});

tracking.ColorTracker.registerColor('Gelb', function(r, g, b) {
if (r > 170 && g > 170 && b < 60) {
return true;
}
return false;
});

tracking.ColorTracker.registerColor('Gruen', function(r, g, b) {
if (r > 10 && r < 30 && g > 50 && g < 180 && b > 30 && b < 70) {
return true;
}
return false;
});

tracking.ColorTracker.registerColor('Rot', function(r, g, b) {
if (r > 130 && r < 220 && g > 30 && g < 120 && b > 20 && b < 80) {
return true;
}
return false;
});

/////////////////////////////// Implementing Sounds with Pizzicato.js //////////////////////////////

var YellowKick = new Pizzicato.Sound('audio/HfG-Franklin-Beat-KickXL.mp3', function() {
//YellowKick.play();
});

var RedBass = new Pizzicato.Sound('audio/HfG-Franklin-Beat-BassXL.mp3', function() {
//RedBass.play();
});

var BlueClap = new Pizzicato.Sound('audio/HfG-Franklin-Beat-ClapXL.mp3', function() {
//BlueClap.play();
});

var GreenPiano = new Pizzicato.Sound('audio/HfG-Franklin-Beat-SnakesXL.mp3', function() {
//GreenPiano.play();
});

// TRACKING COLORS
/////////////////////////// Tracking Gelb ////////////////////////////////////////////////////////

var trackerYe = new tracking.ColorTracker(['Gelb']);
trackerYe.setMinDimension(5);

tracking.track('#video', trackerYe, {
camera: true
});

trackerYe.on('track', function(event) {
var data = event.data;
event.data.forEach(function(rectYe) {
ctx.fillStyle = "white";

for (var i = 20; i < 30; i++) {
ctx.fillRect(Math.random() * 100 + rectYe.x, Math.random() * 100 + rectYe.y, rectYe.width / i, rectYe.height / i);
}

yellowX = rectYe.x;
yellowY = rectYe.y;

});
if (data.length > 0) {
YellowKick.play();
} else {
YellowKick.stop();
}
});

/////////////////////////// Tracking Rot /////////////////////////////////////////////////////////

var trackerR = new tracking.ColorTracker(['Rot']);
trackerR.setMinDimension(5);

tracking.track('#video', trackerR, {
camera: true
});

trackerR.on('track', function(event) {
var data = event.data;

event.data.forEach(function(rectR) {
redX = rectR.x;
redY = rectR.y;

});
if (data.length > 0) {
RedBass.play();
} else {
RedBass.stop();
}
});

/////////////////////////// Tracking Blau ////////////////////////////////////////////////////////

var trackerB = new tracking.ColorTracker(['Blau']);
trackerB.setMinDimension(5);

tracking.track('#video', trackerB, {
camera: true
});

trackerB.on('track', function(event) {
var data = event.data;
event.data.forEach(function(rectB) {
blueX = rectB.x;
blueY = rectB.y;

});
if (data.length > 0) {
BlueClap.play();
} else {
BlueClap.stop();
}
});

/////////////////////////// Tracking Grün ////////////////////////////////////////////////////////

var trackerG = new tracking.ColorTracker(['Gruen']);
trackerG.setMinDimension(5);

tracking.track('#video', trackerG, {
camera: true
});

trackerG.on('track', function(event) {
var data = event.data;
event.data.forEach(function(rectG) {
greenX = rectG.x;
greenY = rectG.y;

});
if (data.length > 0) {
GreenPiano.play();
} else {
GreenPiano.stop();
}
});

// GRID
/////////////////////////// Grid /////////////////////////////////////////////////////////////////

gridSizeX = canvas.width - 2 * border;
gridSizeY = canvas.height - 2 * border;
gapX = gridSizeX / (xCount - 1);
gapY = gridSizeY / (yCount - 1);

var strength = 1; // Strength of Bond
var drag = 2; // Drag on String
var rest = 0; // Rest length of string

mouseParticle = physics.makeParticle();
mouseParticle.makeFixed();

mouseParticle2 = physics.makeParticle();
mouseParticle2.makeFixed();

mouseParticle3 = physics.makeParticle();
mouseParticle3.makeFixed();

mouseParticle4 = physics.makeParticle();
mouseParticle4.makeFixed();

mouseParticle5 = physics.makeParticle();
mouseParticle5.makeFixed();

for (var y = 0; y < yCount; y++) {
for (var x = 0; x < xCount; x++) {
var xPos = x * gapX + border;
var yPos = y * gapY + border;

var myCurrentParticle = physics.makeParticle(5, xPos, yPos); //Länge Strings
myParticles.push(myCurrentParticle);
//physics.makeAttraction(mouseParticle, myCurrentParticle, 1000, 20);
physics.makeAttraction(mouseParticle, myCurrentParticle, -10000, 20);
physics.makeAttraction(mouseParticle2, myCurrentParticle, -10000, 20);
physics.makeAttraction(mouseParticle3, myCurrentParticle, -10000, 20);
physics.makeAttraction(mouseParticle4, myCurrentParticle, -10000, 20);

var myCurrentOrgParticle = physics.makeParticle(20, xPos, yPos);
myCurrentOrgParticle.makeFixed();
myOrgParticles.push(myCurrentOrgParticle);

var myCurrentSpring = physics.makeSpring(myCurrentParticle, myCurrentOrgParticle, strength, drag, rest);
mySprings.push(myCurrentSpring);
}
}

physics.onUpdate().play();

draw();
console.log("Läuft!");
}

// DRAW
/////////////////////////// Function Draw //////////////////////////////////////////////////////////

function draw() {
var thisFrameTime = (thisLoop = new Date) - lastLoop;

// Background
ctx.fillStyle = "black";
ctx.fillRect(0, 0, canvas.width, canvas.height);

// Partikel und Springs
var indexA;
for (var y = 0; y < yCount; y++) {
for (var x = 0; x < xCount; x++) {
indexA = y * xCount + x; // 2D to 1D transformation
var xPosA = myParticles[indexA].position.x;
var yPosA = myParticles[indexA].position.y;
var xPosB = myOrgParticles[indexA].position.x;
var yPosB = myOrgParticles[indexA].position.y;
var col2 = 'rgba(255, 255, 255)';
drawLine(ctx, xPosA, yPosA, xPosB, yPosB, col2);

}
}

var colGelb = 'rgba(255, 255, 0, 20)';
mouseParticle.position.set(yellowX, yellowY);
drawParticle(ctx, mouseParticle.position.x, mouseParticle.position.y, 5, colGelb);

var colRot = 'rgba(255, 0, 0, 20)';
mouseParticle3.position.set(redX, redY);
drawParticle(ctx, mouseParticle3.position.x3, mouseParticle3.position.y3, 5, colRot);

var colBlau = 'rgba(0, 0, 255, 20)';
mouseParticle4.position.set(blueX, blueY);
drawParticle(ctx, mouseParticle4.position.x4, mouseParticle4.position.y4, 5, colBlau);

var colGruen = 'rgba(0, 255, 0, 20)';
mouseParticle2.position.set(greenX, greenY);
drawParticle(ctx, mouseParticle2.position.x2, mouseParticle2.position.y2, 5, colGruen);

// Frame Rate berechnen
frameTime += (thisFrameTime - frameTime) / filterStrength;
lastLoop = thisLoop;
var fpsOut = document.getElementById('frameRate');
fpsOut.innerHTML = "current frame = " + frameCounter + " currente frame rate = " + (100000 / frameTime).toFixed(1) + " fps";
frameCounter += 0.1;
requestAnimFrame(draw);
}

/////////////////////////// Grid ///////////////////////////////////////////////////////////////////

function drawParticle(ctx, x, y, r, col) {

ctx.fillStyle = "rgb(255,255,0)";
ctx.beginPath();
ctx.arc(yellowX, yellowY, r, 400, Math.PI * 2, true);
ctx.closePath();
ctx.fill();

ctx.fillStyle = "rgb(255,0,0)";
ctx.beginPath();
ctx.arc(redX, redY, r, 400, Math.PI * 2, true);
ctx.closePath();
ctx.fill();

ctx.fillStyle = "rgb(0,0,255)";
ctx.beginPath();
ctx.arc(blueX, blueY, r, 400, Math.PI * 2, true);
ctx.closePath();
ctx.fill();

ctx.fillStyle = "rgb(0,255,0)";
ctx.beginPath();
ctx.arc(greenX, greenY, r, 400, Math.PI * 2, true);
ctx.closePath();
ctx.fill();

}

function drawLine(ctx, x1, y1, x2, y2, col) {

ctx.strokeStyle = col;
ctx.beginPath();
ctx.moveTo(x1, y1);
ctx.lineTo(x2, y2);
ctx.stroke();
}

// für Events
function pick(event) {
mouseX = event.layerX;
mouseY = event.layerY;
}

// Animation Request

window.requestAnimFrame = (function() {
return window.requestAnimationFrame ||
window.webkitRequestAnimationFrame ||
window.mozRequestAnimationFrame ||
window.oRequestAnimationFrame ||
window.msRequestAnimationFrame ||
function( /* function */ callback, /* DOMElement */ element) {
window.setTimeout(callback, 1000 / 60);
};
})();

‍