I have a microservice running on AWS Fargate, i.e. as a docker container, that starts slowing down after an hour, but memory usage stays constant. The only easy way to currently get data out from the machine is to either write it to Amazon S3 or stdout.
My question: Is there a package that could somehow sample profiling information that I could just print to stdout?
You could create a Heap and CPU Profile every 15 minutes and upload that to S3 using an instance profile that has permissions to upload to your bucket.
func startCpuProfile() {
cpuProfile, err := os.Create("cpu_profile")
if err != nil {
log.Println("could not create CPU profile: ", err)
}
defer cpuProfile.Close()
if err := pprof.StartCPUProfile(cpuProfile); err != nil {
log.Println("could not start CPU profile: ", err)
}
}
func generateProfileBundle() {
ts := time.Now().Format("2006-01-02_15:04:05")
heapProfile, err := os.Create("heap_profile_" + ts)
if err != nil {
log.Println("could not create memory profile: ", err)
}
defer heapProfile.Close()
runtime.GC()
if err := pprof.WriteHeapProfile(heapProfile); err != nil {
log.Println("could not write memory profile: ", err)
}
pprof.StopCPUProfile()
svc := s3.New(session.New())
cpuFile, err := os.Open("cpu_profile")
_, err = svc.PutObject(&s3.PutObjectInput{
Bucket: aws.String("service_profile_bucket"),
Key: aws.String("cpu_profile_" + ts),
Body: cpuFile,
})
_, err = svc.PutObject(&s3.PutObjectInput{
Bucket: aws.String("service_profile_bucket"),
Key: aws.String("heap_profile_" + ts),
Body: heapProfile,
})
os.Remove("cpu_profile")
os.Remove("heap_profile_" + ts)
startCpuProfile()
}
func main() {
// Start CPU Profile at start up
startCpuProfile()
// Every 15 minutes dump a heap profile, restart the CPU profile and upload to S3
ticker := time.Tick(time.Minute * 15)
go func() {
for _ = range ticker {
generateProfileBundle()
}
}()
}