Service worker: Fetch event listener only works after page reload

I have added a service worker to my page with the code below. It works well once the page has been reloaded and the worker already installed. But does not seem to catch any fetch events before the page is reloaded after I have seen the 'SW INSTALL' log.

app.js

navigator.serviceWorker.register('/worker.js').then((registration) =>
{
    console.log('ServiceWorker registration successful with scope: ', 
registration.scope);
}, (err) =>
{
    console.log('ServiceWorker registration failed: ', err);
});

worker.js

self.addEventListener('install', function (event)
 {
    console.log("SW INSTALL");
 });

self.addEventListener('fetch', function (event)
{
    console.log("FETCHING", event);
    event.respondWith(
            caches.match(event.request)
                    .then(function (response, err)
                            {
                                // Cache hit - return response
                                if (response)
                                {
                                    console.log("FOUND", response, err);
                                    return response;
                                }
                                console.log("MISSED", event.request.mode);
                                return fetch(event.request)
                            }
                    )
    );
});