In order to handle datasets beyond the toy dataset in your post, I put everything for creating an animation into the function animate. Since I'm not familiar with the dataset you are working with (for example I don't know what Rotation does), the script below is the best I can do.
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
from matplotlib import animation
df = pd.DataFrame({
'Time' : [1,1,1,1,2,2,2,2,3,3,3,3],
'id' : ['A','B','C','D','A','B','C','D','A','B','C','D'],
'X1' : [1,8,0,-5,1,1,-6,0,1,8,0,-5],
'Y1' : [-5,2,-5,2,5,-5,-2,2,-5,2,-5,2],
'X2' : [0,0,0,0,-1,-1,-1,-1,0,0,0,0],
'Y2' : [0,0,0,0,1,1,1,1,1,1,1,1],
'Rotation' : [0,0,0,0,-0.78,-0.78,-0.78,-0.78,1.57,1.57,1.57,1.57],
'Angle' : [0,0,0,0,-45,-45,-45,-45,90,90,90,90],
})
# the scattered points in the compass
points_x = np.array(df.groupby(['Time'])['X1'].apply(list))
points_y = np.array(df.groupby(['Time'])['Y1'].apply(list))
# the center of the compass
moving_x = np.array(df.groupby(['Time'])['X2'].apply(list))
moving_y = np.array(df.groupby(['Time'])['Y2'].apply(list))
radius = df.drop_duplicates(subset = ['Time','X2', 'Y2'])[['X2', 'Y2']].values
angles = df['Angle'].unique()
rot_mat = lambda theta:np.array([
[np.cos(np.deg2rad(theta)),-np.sin(np.deg2rad(theta))],
[np.sin(np.deg2rad(theta)),np.cos(np.deg2rad(theta))]
])
fig, ax = plt.subplots(figsize = (8,8))
def animate(i) :
ax.clear()
ax.set_xlim(-20,20)
ax.set_ylim(-20,20)
points = ax.scatter(points_x[i]+radius[i][0], points_y[i]+radius[i][1], c = 'blue', marker = '*')
moving_point = ax.scatter(moving_x[i],moving_y[i], c = 'black', marker = 'x')
circle = plt.Circle(radius[i], 10, color = 'black', fill = False)
ax.add_patch(circle)
ends_one = np.array([[-7.07106781,7.07106781],[7.07106781,-7.07106781]])
ends_two = np.array([[-7.07106781,-7.07106781],[7.07106781,7.07106781]])
ends_one = ends_one @ rot_mat(angles[i]) + radius[i]
ends_two = ends_two @ rot_mat(angles[i]) + radius[i]
line1, = ax.plot([], [], ls='--', color='black',lw=1, zorder=10,animated=True)
line2, = ax.plot([], [], ls='--', color='black',lw=1, zorder=10,animated=True)
line1.set_data([ends_one[0][0],ends_one[1][0]],[ends_one[0][1],ends_one[1][1]])
line2.set_data([ends_two[0][0],ends_two[1][0]],[ends_two[0][1],ends_two[1][1]])
tags = ['N','E','S','W']
tag_pos = np.array([[0,8.5],[8.5,0],[0,-8.5],[-8.5,0]])
tag_pos = tag_pos @ rot_mat(angles[i])
for tag,pos in zip(tags,tag_pos):
ax.annotate(tag,xy=pos+radius[i], xycoords='data',
fontsize=10,horizontalalignment='right', verticalalignment='bottom')
ani = animation.FuncAnimation(fig, animate, np.arange(0,3), blit = False)
ani.save('test.gif', writer='pillow', fps=3)
As you can see, coordinates can be passed iteratively into the function animate to generate frames of a compass. After you run this script, you will get a gif file test.gif, which can be previewed

Remove text of past frames
There are two ways you can do this
1. If you are using a function to create animation
put ax.clear() at the beginning of the function, for example
import matplotlib.pyplot as plt
from matplotlib import animation
rot_mat = lambda theta:np.array([
[np.cos(np.deg2rad(theta)),-np.sin(np.deg2rad(theta))],
[np.sin(np.deg2rad(theta)),np.cos(np.deg2rad(theta))]
])
fig, ax = plt.subplots(figsize = (8,8))
centers = np.linspace(1,36,36).reshape(18,2)*0.2
angles = np.linspace(45,90,18)
def animate(i) :
ax.clear()
ax.set_xlim(-20,20)
ax.set_ylim(-20,20)
circle = plt.Circle(centers[i], 10, color = 'black', fill = True)
circle.set_facecolor('violet')
ax.add_patch(circle)
tags = ['N','E','S','W']
tag_pos = np.array([[0,8.5],[8.5,0],[0,-8.5],[-8.5,0]])
tag_pos = tag_pos @ rot_mat(angles[i])
for tag,pos in zip(tags,tag_pos):
ax.annotate(tag,xy=pos+centers[i], xycoords='data',
fontsize=10,horizontalalignment='right', verticalalignment='bottom')
ani = animation.FuncAnimation(fig, animate, np.arange(0,18), blit = False)
ani.save('test.gif', writer='pillow', fps=18)
This will create a gif like this

2. If you are not using a function to create animation
You need to manually remove the text, for example
import matplotlib.pyplot as plt
from matplotlib import animation
rot_mat = lambda theta:np.array([
[np.cos(np.deg2rad(theta)),-np.sin(np.deg2rad(theta))],
[np.sin(np.deg2rad(theta)),np.cos(np.deg2rad(theta))]
])
centers = np.linspace(1,36,36).reshape(18,2)*0.2
angles = np.linspace(45,90,18)
fig, ax = plt.subplots(figsize = (8,8))
ax.set_xlim(-20,20)
ax.set_ylim(-20,20)
circle = plt.Circle(centers[0], 10, color = 'black', fill = True)
circle.set_facecolor('violet')
ax.add_patch(circle)
annotates = []
tags = ['N','E','S','W']
tag_pos = np.array([[0,8.5],[8.5,0],[0,-8.5],[-8.5,0]])
tag_pos = tag_pos @ rot_mat(angles[0])
for tag,pos in zip(tags,tag_pos):
ann = ax.annotate(tag,xy=pos+centers[0], xycoords='data',
fontsize=10,horizontalalignment='right', verticalalignment='bottom')
annotates.append(ann)
for i in range(18):
circle.center = centers[i]
# remove the previous text
for item in annotates: item.remove()
annotates = []
tag_pos = tag_pos @ rot_mat(angles[i])
for tag,pos in zip(tags,tag_pos):
ann = ax.annotate(tag,xy=pos+centers[i], xycoords='data',
fontsize=10,horizontalalignment='right', verticalalignment='bottom')
annotates.append(ann)
ani = animation.FuncAnimation(fig, animate, np.arange(0,18), blit = False)
ani.save('test.gif', writer='pillow', fps=18)
as you can see, with this line of code for item in annotates: item.remove(), the text of a previous frame are removed. Therefore, by running this script, you will also get
