AngularJS input ng-model not updating

I am trying to create a simple pagination directive with an isolated scope. For some reason when I manually change the value it gets a bit finnicky. Here is my problem:

When I page forward and backward, it works great. Awesome

When I enter a page into the field it works. Great

However, if I enter a page into the field and then try to go forward and backward, the ng-model seems to break after I enter a page into the field. I had it working when I did not isolate my scope but I am confused as to why it would break it. Here is my code:

HTML:

<paginate go-to-page="goToPage(page)" total-pages="results.hits.pages" total-hits="results.hits.total"></paginate>

Directive:

'use strict';

angular.module('facet.directives')
    .directive('paginate', function(){
        return {
            restrict: 'E',
            template: '<div class="pull-right" ng-if="(totalPages !== undefined) && (totalPages > 0)">'+
                '<span class="left-caret hoverable" ng-click="changePage(current-1)" ng-show="current > 1"></span>&nbsp;&nbsp;Page'+
                '&nbsp;&nbsp;&nbsp;<input type="number" ng-model="current" class="pagination-input" ng-keypress="enterPage($event)"/> of'+
                '&nbsp;&nbsp;&nbsp;{{totalPages}}&nbsp;&nbsp;'+
                '<span class="right-caret hoverable" ng-click="changePage(current+1)" ng-show="current < totalPages"></span>'+
            '</div>',
            scope: {
                goToPage: '&',
                totalPages: '=',
                totalHits: '='
            },
            link: function(scope) {
                scope.current = 1;
                scope.changePage = function(page) {
                    scope.current = page;
                    window.scrollTo(0,0);
                    scope.goToPage({page:page});
                };
                scope.enterPage = function(event) {
                    if(event.keyCode == 13) {
                        scope.changePage(scope.current);
                    }
                }
            }
        }
    });

What am I doing wrong?