Earlier my files were uploading in the storage folder. But now I want to upload images on the s3 bucket. how can I migrate my existing local data on the s3 bucket?
I found the script here https://www.stefanwienert.de/blog/2018/11/05/active-storage-migrate-between-providers-from-local-to-amazon/ But getting an error of
NoMethodError (private method `open' called for Active Storage
So what should I do for migrating my local data to the s3 bucket?
Is there in a simpler way?
Hi I also got that error and I have changed the script to rake task like below:
# frozen_string_literal: true
namespace :active_storage do
desc 'Migrate ActiveStorage files from local to Amazon S3'
task migrate: :environment do
module ActiveStorage
class Downloader
def initialize(blob, tempdir: nil)
@blob = blob
@tempdir = tempdir
end
def download_blob_to_tempfile
open_tempfile do |file|
download_blob_to file
verify_integrity_of file
yield file
end
end
private
attr_reader :blob, :tempdir
def open_tempfile
file = Tempfile.open(["ActiveStorage-#{blob.id}-", blob.filename.extension_with_delimiter], tempdir)
begin
yield file
ensure
file.close!
end
end
def download_blob_to(file)
file.binmode
blob.download { |chunk| file.write(chunk) }
file.flush
file.rewind
end
def verify_integrity_of(file)
raise ActiveStorage::IntegrityError unless Digest::MD5.file(file).base64digest == blob.checksum
end
end
end
module AsDownloadPatch
def open(tempdir: nil, &block)
ActiveStorage::Downloader.new(self, tempdir: tempdir).download_blob_to_tempfile(&block)
end
end
Rails.application.config.to_prepare do
ActiveStorage::Blob.send(:include, AsDownloadPatch)
end
def migrate(from, to)
configs = Rails.configuration.active_storage.service_configurations
from_service = ActiveStorage::Service.configure from, configs
to_service = ActiveStorage::Service.configure to, configs
ActiveStorage::Blob.service = from_service
puts "#{ActiveStorage::Blob.count} Blobs to go..."
ActiveStorage::Blob.find_each do |blob|
print '.'
file = Tempfile.new("file#{Time.now}")
file.binmode
file << blob.download
file.rewind
checksum = blob.checksum
to_service.upload(blob.key, file, checksum: checksum)
rescue Errno::ENOENT
puts 'Rescued by Errno::ENOENT statement.'
next
end
end
migrate(:local, :s3)
end
end
A much simpler way that works:
amazon section to your config/storage.ymllike
change your storage service to :amazon in your config/environments/production.rb
add this rake task as lib/tasks/storage.rake
namespace :storage do
task reupload: :environment do
[User, Event].each do |clazz|
collection = clazz.with_attached_image
puts "#{clazz} has #{collection.count} images"
collection.find_each do |user|
next unless user.image.blob
user
.image
.blob
.open do |f|
user.image.attach(io: f, filename: user.image.blob.filename)
end
print "."
end
puts
end
end
end
run and test locally to try it with rake storage:reupload (and changing to config.active_storage.service = :amazon in config/environments/development.rb)
check that everything works fine locally (your images should link to your AWS URLs)
upload to your server (cap deploy for instance)
run RAILS_ENV=production bundle exec rake storage:reupload in the directory of your project
wait for a bit, depending on how many images you are reuploading
Profit! Enjoy! Party time!
Pros
Cons
has_many_attached but shouldn't be too many changesimage (nothing too hard to fix)namespace :active_storage do
desc 'Migrate ActiveStorage files from local to minio'
task migrate: :environment do
module ActiveStorage
class Downloader
def initialize(blob, tempdir: nil)
@blob = blob
@tempdir = tempdir
end
def download_blob_to_tempfile
open_tempfile do |file|
download_blob_to file
verify_integrity_of file
yield file
end
end
private
attr_reader :blob, :tempdir
def open_tempfile
file = Tempfile.open(["ActiveStorage-#{blob.id}-", blob.filename.extension_with_delimiter], tempdir)
begin
yield file
ensure
file.close!
end
end
def download_blob_to(file)
file.binmode
blob.download { |chunk| file.write(chunk) }
file.flush
file.rewind
end
def verify_integrity_of(file)
raise ActiveStorage::IntegrityError unless Digest::MD5.file(file).base64digest == blob.checksum
end
end
end
module AsDownloadPatch
def open(tempdir: nil, &block)
ActiveStorage::Downloader.new(self, tempdir: tempdir).download_blob_to_tempfile(&block)
end
end
Rails.application.config.to_prepare do
ActiveStorage::Blob.send(:include, AsDownloadPatch)
end
def migrate(from, to)
config_file = Rails.root.join("config/storage.yml")
configs = ActiveSupport::ConfigurationFile.parse(config_file)
from_service = ActiveStorage::Service.configure from, configs
to_service = ActiveStorage::Service.configure to, configs
ActiveStorage::Blob.service = from_service
puts "#{ActiveStorage::Blob.count} Blobs to go..."
ActiveStorage::Blob.find_each do |blob|
print '.'
file = Tempfile.new("file#{Time.now}")
file.binmode
file << blob.download
file.rewind
checksum = blob.checksum
to_service.upload(blob.key, file, checksum: checksum)
rescue Errno::ENOENT
puts 'Rescued by Errno::ENOENT statement.'
next
end
end
migrate(:local, :minio)
end
end
Based on the answer from Dorian I constructed a simpler version. For this version you don't need to select your classes or know/care how the method inside the class is called.
It should also work for has_many_attached (since we start from the attachment itself)
Just like with Dorian's version, you need to add your config for s3 before this and have this deployed
ActiveStorage::Attachment.find_each do |at|
next unless at.blob.service_name == "local"
begin
blob = at.blob
blob.open do |f|
at.record.send(at.name).attach(io: f, content_type: blob.content_type, filename: blob.filename)
end
blob.destroy
rescue ActiveStorage::FileNotFoundError
# Add some message or warning here if you fancy
end
end
Simpler way to migrate your local files to s3-service is:
— setup as mirror it first,
— sync all mirrors with rake,
— remove mirror setup,
— remove local files after check
I guess this solution will help: How to sync new ActiveStorage mirrors?